From b7814bb942a57b126aa81067dbcc3feae736ad73 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 4 Nov 2024 11:45:36 +0100 Subject: [PATCH 01/22] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Changing=20=3D=3D=20?= =?UTF-8?q?by=20is=5F=20in=20sqlalchemy=20queries=20(#6654)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/maintenance/migrate_project/src/db.py | 4 ++-- .../simcore_service_webserver/studies_dispatcher/_catalog.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/maintenance/migrate_project/src/db.py b/scripts/maintenance/migrate_project/src/db.py index 6da7430a834..2d4fddfcd37 100644 --- a/scripts/maintenance/migrate_project/src/db.py +++ b/scripts/maintenance/migrate_project/src/db.py @@ -48,7 +48,7 @@ def _get_project(connection: Connection, project_uuid: UUID) -> ResultProxy: def _get_hidden_project(connection: Connection, prj_owner: int) -> ResultProxy: return connection.execute( select(projects).where( - and_(projects.c.prj_owner == prj_owner, projects.c.hidden == True) + and_(projects.c.prj_owner == prj_owner, projects.c.hidden.is_(True)) ) ) @@ -61,7 +61,7 @@ def _get_file_meta_data_without_soft_links( and_( file_meta_data.c.node_id == f"{node_uuid}", file_meta_data.c.project_id == f"{project_id}", - file_meta_data.c.is_soft_link != True, + file_meta_data.c.is_soft_link.is_not(True), ) ) ) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py index 3df62ebd379..b7144d6725b 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py @@ -97,7 +97,7 @@ async def iter_latest_product_services( ) & (services_meta_data.c.deprecated.is_(None)) & (services_access_rights.c.gid == EVERYONE_GROUP_ID) - & (services_access_rights.c.execute_access == True) + & (services_access_rights.c.execute_access.is_(True)) & (services_access_rights.c.product_name == product_name) ) ) @@ -161,7 +161,7 @@ async def validate_requested_service( sa.select(services_consume_filetypes.c.is_guest_allowed) .where( (services_consume_filetypes.c.service_key == service_key) - & (services_consume_filetypes.c.is_guest_allowed == True) + & (services_consume_filetypes.c.is_guest_allowed.is_(True)) ) .limit(1) ) From 24445c0c9c589922a91dec014e5fdd39f111b2b2 Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:55:37 +0100 Subject: [PATCH 02/22] =?UTF-8?q?=F0=9F=90=9B=20[Frontend]=20Fix=20Service?= =?UTF-8?q?=20browser=20(#6659)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/source/class/osparc/store/Services.js | 5 +++-- .../client/source/class/osparc/utils/Utils.js | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/store/Services.js b/services/static-webserver/client/source/class/osparc/store/Services.js index 569f5ce08e4..f6851b3aa43 100644 --- a/services/static-webserver/client/source/class/osparc/store/Services.js +++ b/services/static-webserver/client/source/class/osparc/store/Services.js @@ -59,7 +59,7 @@ qx.Class.define("osparc.store.Services", { let serviceLatest = servicesLatest[key]; if (excludeFrontend && key.includes("simcore/services/frontend/")) { // do not add frontend services - return; + continue; } if (excludeDeprecated && serviceLatest["retired"]) { // first check if a previous version of this service isn't retired @@ -68,13 +68,14 @@ qx.Class.define("osparc.store.Services", { for (let j=0; j Date: Mon, 4 Nov 2024 17:49:47 +0100 Subject: [PATCH 03/22] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20introduce=20webserve?= =?UTF-8?q?r=204=20tests=20(#6663)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci-testing-deploy.yml | 46 +++++++++++++++++++ .../server/tests/unit/with_dbs/04/conftest.py | 15 ++++++ .../with_dbs/{01 => 04}/folders/conftest.py | 0 .../{01 => 04}/folders/test_folders.py | 0 .../test_resource_manager.py | 0 ...fications__db_comp_tasks_listening_task.py | 0 .../with_dbs/{01 => 04}/products/conftest.py | 0 .../{01 => 04}/products/test_products_db.py | 0 .../products/test_products_handlers.py | 0 .../{01 => 04}/products/test_products_rpc.py | 0 .../{01 => 04}/studies_dispatcher/conftest.py | 0 .../test_studies_dispatcher_handlers.py | 0 .../test_studies_dispatcher_projects.py | 0 .../test_studies_dispatcher_studies_access.py | 0 .../with_dbs/{01 => 04}/wallets/conftest.py | 0 .../{01 => 04}/wallets/payments/conftest.py | 0 .../wallets/payments/test_payments.py | 0 .../wallets/payments/test_payments_methods.py | 0 .../wallets/payments/test_payments_rpc.py | 0 .../{01 => 04}/wallets/test_wallets.py | 0 .../{01 => 04}/wallets/test_wallets_groups.py | 0 .../{01 => 04}/workspaces/conftest.py | 0 .../{01 => 04}/workspaces/test_workspaces.py | 0 ...t_workspaces__folders_and_projects_crud.py | 0 ...t_workspaces__list_projects_full_search.py | 0 ...ces__moving_projects_between_workspaces.py | 0 .../workspaces/test_workspaces_groups.py | 0 27 files changed, 61 insertions(+) create mode 100644 services/web/server/tests/unit/with_dbs/04/conftest.py rename services/web/server/tests/unit/with_dbs/{01 => 04}/folders/conftest.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/folders/test_folders.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/garbage_collector/test_resource_manager.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/notifications/test_notifications__db_comp_tasks_listening_task.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/products/conftest.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/products/test_products_db.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/products/test_products_handlers.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/products/test_products_rpc.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/studies_dispatcher/conftest.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/studies_dispatcher/test_studies_dispatcher_handlers.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/studies_dispatcher/test_studies_dispatcher_projects.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/studies_dispatcher/test_studies_dispatcher_studies_access.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/wallets/conftest.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/wallets/payments/conftest.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/wallets/payments/test_payments.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/wallets/payments/test_payments_methods.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/wallets/payments/test_payments_rpc.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/wallets/test_wallets.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/wallets/test_wallets_groups.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/workspaces/conftest.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/workspaces/test_workspaces.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/workspaces/test_workspaces__folders_and_projects_crud.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/workspaces/test_workspaces__list_projects_full_search.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/workspaces/test_workspaces__moving_projects_between_workspaces.py (100%) rename services/web/server/tests/unit/with_dbs/{01 => 04}/workspaces/test_workspaces_groups.py (100%) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 38163963d6d..34f17a07d85 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -450,6 +450,51 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + unit-test-webserver-04: + needs: changes + if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }} + timeout-minutes: 25 # if this timeout gets too small, then split the tests + name: "[unit] webserver 04" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.11"] + os: [ubuntu-22.04] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + - name: setup python environment + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: install uv + uses: astral-sh/setup-uv@v3 + with: + version: "0.4.x" + enable-cache: false + cache-dependency-glob: "**/web/server/requirements/ci.txt" + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - name: install webserver + run: ./ci/github/unit-testing/webserver.bash install + - name: test + run: ./ci/github/unit-testing/webserver.bash test_with_db 04 + - uses: codecov/codecov-action@v4.6.0 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: unittests #optional + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + unit-test-storage: needs: changes if: ${{ needs.changes.outputs.storage == 'true' || github.event_name == 'push' }} @@ -1875,6 +1920,7 @@ jobs: unit-test-webserver-01, unit-test-webserver-02, unit-test-webserver-03, + unit-test-webserver-04, ] runs-on: ubuntu-latest steps: diff --git a/services/web/server/tests/unit/with_dbs/04/conftest.py b/services/web/server/tests/unit/with_dbs/04/conftest.py new file mode 100644 index 00000000000..0d136f20637 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/conftest.py @@ -0,0 +1,15 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +import pytest + + +@pytest.fixture +def app_environment( + app_environment: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> dict[str, str]: + # NOTE: overrides app_environment + monkeypatch.setenv("WEBSERVER_GARBAGE_COLLECTOR", "null") + return app_environment | {"WEBSERVER_GARBAGE_COLLECTOR": "null"} diff --git a/services/web/server/tests/unit/with_dbs/01/folders/conftest.py b/services/web/server/tests/unit/with_dbs/04/folders/conftest.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/folders/conftest.py rename to services/web/server/tests/unit/with_dbs/04/folders/conftest.py diff --git a/services/web/server/tests/unit/with_dbs/01/folders/test_folders.py b/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/folders/test_folders.py rename to services/web/server/tests/unit/with_dbs/04/folders/test_folders.py diff --git a/services/web/server/tests/unit/with_dbs/01/garbage_collector/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/garbage_collector/test_resource_manager.py rename to services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py diff --git a/services/web/server/tests/unit/with_dbs/01/notifications/test_notifications__db_comp_tasks_listening_task.py b/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/notifications/test_notifications__db_comp_tasks_listening_task.py rename to services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py diff --git a/services/web/server/tests/unit/with_dbs/01/products/conftest.py b/services/web/server/tests/unit/with_dbs/04/products/conftest.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/products/conftest.py rename to services/web/server/tests/unit/with_dbs/04/products/conftest.py diff --git a/services/web/server/tests/unit/with_dbs/01/products/test_products_db.py b/services/web/server/tests/unit/with_dbs/04/products/test_products_db.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/products/test_products_db.py rename to services/web/server/tests/unit/with_dbs/04/products/test_products_db.py diff --git a/services/web/server/tests/unit/with_dbs/01/products/test_products_handlers.py b/services/web/server/tests/unit/with_dbs/04/products/test_products_handlers.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/products/test_products_handlers.py rename to services/web/server/tests/unit/with_dbs/04/products/test_products_handlers.py diff --git a/services/web/server/tests/unit/with_dbs/01/products/test_products_rpc.py b/services/web/server/tests/unit/with_dbs/04/products/test_products_rpc.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/products/test_products_rpc.py rename to services/web/server/tests/unit/with_dbs/04/products/test_products_rpc.py diff --git a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/conftest.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/studies_dispatcher/conftest.py rename to services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py diff --git a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_handlers.py rename to services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py diff --git a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_projects.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_projects.py rename to services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py diff --git a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_studies_access.py rename to services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/conftest.py b/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/wallets/conftest.py rename to services/web/server/tests/unit/with_dbs/04/wallets/conftest.py diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/payments/conftest.py b/services/web/server/tests/unit/with_dbs/04/wallets/payments/conftest.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/wallets/payments/conftest.py rename to services/web/server/tests/unit/with_dbs/04/wallets/payments/conftest.py diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments.py b/services/web/server/tests/unit/with_dbs/04/wallets/payments/test_payments.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments.py rename to services/web/server/tests/unit/with_dbs/04/wallets/payments/test_payments.py diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_methods.py b/services/web/server/tests/unit/with_dbs/04/wallets/payments/test_payments_methods.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_methods.py rename to services/web/server/tests/unit/with_dbs/04/wallets/payments/test_payments_methods.py diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_rpc.py b/services/web/server/tests/unit/with_dbs/04/wallets/payments/test_payments_rpc.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_rpc.py rename to services/web/server/tests/unit/with_dbs/04/wallets/payments/test_payments_rpc.py diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/test_wallets.py b/services/web/server/tests/unit/with_dbs/04/wallets/test_wallets.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/wallets/test_wallets.py rename to services/web/server/tests/unit/with_dbs/04/wallets/test_wallets.py diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/test_wallets_groups.py b/services/web/server/tests/unit/with_dbs/04/wallets/test_wallets_groups.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/wallets/test_wallets_groups.py rename to services/web/server/tests/unit/with_dbs/04/wallets/test_wallets_groups.py diff --git a/services/web/server/tests/unit/with_dbs/01/workspaces/conftest.py b/services/web/server/tests/unit/with_dbs/04/workspaces/conftest.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/workspaces/conftest.py rename to services/web/server/tests/unit/with_dbs/04/workspaces/conftest.py diff --git a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces.py rename to services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py diff --git a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__folders_and_projects_crud.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__folders_and_projects_crud.py rename to services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py diff --git a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__list_projects_full_search.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__list_projects_full_search.py rename to services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py diff --git a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__moving_projects_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__moving_projects_between_workspaces.py rename to services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py diff --git a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces_groups.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces_groups.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces_groups.py rename to services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces_groups.py From dac26e1915a946e07f0ce1ba153be09a2918da70 Mon Sep 17 00:00:00 2001 From: Matus Drobuliak <60785969+matusdrobuliak66@users.noreply.github.com> Date: Mon, 4 Nov 2024 18:36:51 +0100 Subject: [PATCH 04/22] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactor=20RUT=20(re?= =?UTF-8?q?naming/rearranging/error=20handling)=20(#6648)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resource_usage_tracker/errors.py | 9 - .../core/settings.py | 2 +- .../resource-usage-tracker/docker/boot.sh | 2 +- .../api/rest/_health.py | 2 +- .../api/rest/_resource_tracker.py | 16 +- .../api/rest/dependencies.py | 2 +- .../api/rpc/_resource_tracker.py | 37 +- .../api/rpc/routes.py | 2 +- .../core/application.py | 51 +- .../core/errors.py | 20 - .../{modules => exceptions}/__init__.py | 0 .../exceptions/errors.py | 70 +++ .../exceptions/handlers/__init__.py | 20 + .../exceptions/handlers/_http_error.py | 61 +++ .../{web_main.py => main.py} | 0 ...transactions.py => credit_transactions.py} | 0 ...cker_pricing_plans.py => pricing_plans.py} | 0 ...ng_unit_costs.py => pricing_unit_costs.py} | 0 ...cker_pricing_units.py => pricing_units.py} | 0 ...racker_service_runs.py => service_runs.py} | 0 .../modules/prometheus.py | 73 --- ...ckground_task_periodic_heartbeat_check.py} | 10 +- ...und_task_periodic_heartbeat_check_setup.py | 74 +++ ...transactions.py => credit_transactions.py} | 8 +- .../services/modules/__init__.py | 0 .../{ => services}/modules/db/__init__.py | 0 .../modules/db/repositories/__init__.py | 0 .../modules/db/repositories/_base.py | 0 .../db/repositories/resource_tracker.py | 66 ++- .../{ => services}/modules/rabbitmq.py | 2 +- .../{ => services}/modules/redis.py | 2 +- .../{ => services}/modules/s3.py | 2 +- ...cker_pricing_plans.py => pricing_plans.py} | 17 +- ...cker_pricing_units.py => pricing_units.py} | 2 +- .../process_message_running_service.py} | 6 +- .../process_message_running_service_setup.py} | 23 +- ...racker_service_runs.py => service_runs.py} | 9 +- .../utils.py} | 0 .../tests/unit/api_rest/test_api_meta.py | 4 +- .../tests/unit/conftest.py | 1 + .../tests/unit/modules/test_rabbitmq.py | 4 +- .../tests/unit/modules/test_redis.py | 6 +- .../tests/unit/test_computation_of_credits.py | 2 +- .../unit/{test_web_main.py => test_main.py} | 2 +- .../tests/unit/with_dbs/conftest.py | 6 +- .../list_of_prometheus_mocked_outputs.json | 496 ------------------ ...i_resource_tracker_service_runs__export.py | 4 +- ...rce_tracker_service_runs__list_billable.py | 7 +- ...ckground_task_periodic_heartbeat_check.py} | 12 +- .../with_dbs/test_process_rabbitmq_message.py | 4 +- ...t_process_rabbitmq_message_with_billing.py | 4 +- ...ss_rabbitmq_message_with_billing_cost_0.py | 4 +- .../_pricing_plans_admin_handlers.py | 10 +- 53 files changed, 369 insertions(+), 785 deletions(-) delete mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/errors.py delete mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{modules => exceptions}/__init__.py (100%) create mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/errors.py create mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/__init__.py create mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{web_main.py => main.py} (100%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/{resource_tracker_credit_transactions.py => credit_transactions.py} (100%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/{resource_tracker_pricing_plans.py => pricing_plans.py} (100%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/{resource_tracker_pricing_unit_costs.py => pricing_unit_costs.py} (100%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/{resource_tracker_pricing_units.py => pricing_units.py} (100%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/{resource_tracker_service_runs.py => service_runs.py} (100%) delete mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/prometheus.py rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{resource_tracker_background_task.py => services/background_task_periodic_heartbeat_check.py} (94%) create mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check_setup.py rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/{resource_tracker_credit_transactions.py => credit_transactions.py} (89%) create mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/__init__.py rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{ => services}/modules/db/__init__.py (100%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{ => services}/modules/db/repositories/__init__.py (100%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{ => services}/modules/db/repositories/_base.py (100%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{ => services}/modules/db/repositories/resource_tracker.py (96%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{ => services}/modules/rabbitmq.py (97%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{ => services}/modules/redis.py (94%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{ => services}/modules/s3.py (97%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/{resource_tracker_pricing_plans.py => pricing_plans.py} (93%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/{resource_tracker_pricing_units.py => pricing_units.py} (97%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{resource_tracker_process_messages.py => services/process_message_running_service.py} (98%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{resource_tracker.py => services/process_message_running_service_setup.py} (61%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/{resource_tracker_service_runs.py => service_runs.py} (96%) rename services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/{resource_tracker_utils.py => services/utils.py} (100%) rename services/resource-usage-tracker/tests/unit/{test_web_main.py => test_main.py} (75%) delete mode 100644 services/resource-usage-tracker/tests/unit/with_dbs/data/list_of_prometheus_mocked_outputs.json rename services/resource-usage-tracker/tests/unit/with_dbs/{test_background_task.py => test_background_task_periodic_heartbeat_check.py} (95%) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/errors.py deleted file mode 100644 index 44549841802..00000000000 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/errors.py +++ /dev/null @@ -1,9 +0,0 @@ -from pydantic.errors import PydanticErrorMixin - - -class ResourceUsageTrackerRuntimeError(PydanticErrorMixin, RuntimeError): - msg_template: str = "Resource-usage-tracker unexpected error" - - -class CustomResourceUsageTrackerError(ResourceUsageTrackerRuntimeError): - msg_template: str = "Error: {msg}" diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index cf6f636622a..214d51ad11b 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -135,7 +135,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) DY_SIDECAR_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( - default={}, + default_factory=dict, env=["DY_SIDECAR_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/resource-usage-tracker/docker/boot.sh b/services/resource-usage-tracker/docker/boot.sh index d6e54f8f347..d3aa7606d03 100755 --- a/services/resource-usage-tracker/docker/boot.sh +++ b/services/resource-usage-tracker/docker/boot.sh @@ -56,7 +56,7 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then --log-level \"${SERVER_LOG_LEVEL}\" " else - exec uvicorn simcore_service_resource_usage_tracker.web_main:the_app \ + exec uvicorn simcore_service_resource_usage_tracker.main:the_app \ --host 0.0.0.0 \ --log-level "${SERVER_LOG_LEVEL}" fi diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/_health.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/_health.py index c49f8e218dd..720d51b3ad5 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/_health.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/_health.py @@ -7,7 +7,7 @@ from models_library.errors import RABBITMQ_CLIENT_UNHEALTHY_MSG from servicelib.rabbitmq import RabbitMQClient -from ...modules.rabbitmq import get_rabbitmq_client_from_request +from ...services.modules.rabbitmq import get_rabbitmq_client_from_request logger = logging.getLogger(__name__) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/_resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/_resource_tracker.py index c12acad3d49..fc20977e3aa 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/_resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/_resource_tracker.py @@ -12,11 +12,7 @@ ) from models_library.resource_tracker import CreditTransactionId -from ...services import ( - resource_tracker_credit_transactions, - resource_tracker_pricing_plans, - resource_tracker_pricing_units, -) +from ...services import credit_transactions, pricing_plans, pricing_units _logger = logging.getLogger(__name__) @@ -38,9 +34,7 @@ async def get_credit_transactions_sum( wallet_total_credits: Annotated[ WalletTotalCredits, - Depends( - resource_tracker_credit_transactions.sum_credit_transactions_by_product_and_wallet - ), + Depends(credit_transactions.sum_credit_transactions_by_product_and_wallet), ], ): return wallet_total_credits @@ -56,7 +50,7 @@ async def get_credit_transactions_sum( async def create_credit_transaction( transaction_id: Annotated[ CreditTransactionId, - Depends(resource_tracker_credit_transactions.create_credit_transaction), + Depends(credit_transactions.create_credit_transaction), ], ): return {"credit_transaction_id": transaction_id} @@ -77,7 +71,7 @@ async def create_credit_transaction( async def get_service_default_pricing_plan( service_pricing_plans: Annotated[ PricingPlanGet, - Depends(resource_tracker_pricing_plans.get_service_default_pricing_plan), + Depends(pricing_plans.get_service_default_pricing_plan), ], ): return service_pricing_plans @@ -93,7 +87,7 @@ async def get_service_default_pricing_plan( async def get_pricing_plan_unit( pricing_unit: Annotated[ PricingUnitGet, - Depends(resource_tracker_pricing_units.get_pricing_unit), + Depends(pricing_units.get_pricing_unit), ] ): return pricing_unit diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/dependencies.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/dependencies.py index b005829b011..49ce9523cfe 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/dependencies.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/dependencies.py @@ -12,7 +12,7 @@ from servicelib.fastapi.dependencies import get_app, get_reverse_url_mapper from sqlalchemy.ext.asyncio import AsyncEngine -from ...modules.db.repositories._base import BaseRepository +from ...services.modules.db.repositories._base import BaseRepository logger = logging.getLogger(__name__) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py index cae70b1152c..d7e9a5ca74d 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py @@ -26,16 +26,13 @@ from models_library.wallets import WalletID from pydantic import AnyUrl from servicelib.rabbitmq import RPCRouter -from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( - CustomResourceUsageTrackerError, -) from ...core.settings import ApplicationSettings -from ...modules.db.repositories.resource_tracker import ResourceTrackerRepository -from ...modules.s3 import get_s3_client -from ...services import resource_tracker_pricing_plans as pricing_plans -from ...services import resource_tracker_pricing_units as pricing_units -from ...services import resource_tracker_service_runs as service_runs +from ...services import pricing_plans, pricing_units, service_runs +from ...services.modules.db.repositories.resource_tracker import ( + ResourceTrackerRepository, +) +from ...services.modules.s3 import get_s3_client router = RPCRouter() @@ -43,7 +40,7 @@ ## Service runs -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def get_service_run_page( app: FastAPI, *, @@ -69,7 +66,7 @@ async def get_service_run_page( ) -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def export_service_runs( app: FastAPI, *, @@ -98,7 +95,7 @@ async def export_service_runs( ) -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def get_osparc_credits_aggregated_usages_page( app: FastAPI, *, @@ -127,7 +124,7 @@ async def get_osparc_credits_aggregated_usages_page( ## Pricing plans -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def get_pricing_plan( app: FastAPI, *, @@ -141,7 +138,7 @@ async def get_pricing_plan( ) -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def list_pricing_plans( app: FastAPI, *, @@ -153,7 +150,7 @@ async def list_pricing_plans( ) -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def create_pricing_plan( app: FastAPI, *, @@ -165,7 +162,7 @@ async def create_pricing_plan( ) -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def update_pricing_plan( app: FastAPI, *, @@ -182,7 +179,7 @@ async def update_pricing_plan( ## Pricing units -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def get_pricing_unit( app: FastAPI, *, @@ -198,7 +195,7 @@ async def get_pricing_unit( ) -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def create_pricing_unit( app: FastAPI, *, @@ -212,7 +209,7 @@ async def create_pricing_unit( ) -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def update_pricing_unit( app: FastAPI, *, @@ -229,7 +226,7 @@ async def update_pricing_unit( ## Pricing plan to service -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def list_connected_services_to_pricing_plan_by_pricing_plan( app: FastAPI, *, @@ -246,7 +243,7 @@ async def list_connected_services_to_pricing_plan_by_pricing_plan( return output -@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,)) +@router.expose(reraise_if_error_type=()) async def connect_service_to_pricing_plan( app: FastAPI, *, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/routes.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/routes.py index 3cd6220b9e7..c15175e2564 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/routes.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/routes.py @@ -4,7 +4,7 @@ ) from servicelib.rabbitmq import RPCRouter -from ...modules.rabbitmq import get_rabbitmq_rpc_server +from ...services.modules.rabbitmq import get_rabbitmq_rpc_server from . import _resource_tracker ROUTERS: list[RPCRouter] = [ diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py index a97db0170ae..34db96d40ee 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py @@ -2,13 +2,7 @@ from fastapi import FastAPI from servicelib.fastapi.openapi import override_fastapi_openapi_method -from servicelib.fastapi.prometheus_instrumentation import ( - setup_prometheus_instrumentation, -) from servicelib.fastapi.tracing import setup_tracing -from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( - CustomResourceUsageTrackerError, -) from .._meta import ( API_VERSION, @@ -20,12 +14,17 @@ ) from ..api.rest.routes import setup_api_routes from ..api.rpc.routes import setup_rpc_api_routes -from ..modules.db import setup as setup_db -from ..modules.rabbitmq import setup as setup_rabbitmq -from ..modules.redis import setup as setup_redis -from ..modules.s3 import setup as setup_s3 -from ..resource_tracker import setup as setup_resource_tracker -from .errors import http404_error_handler +from ..exceptions.handlers import setup_exception_handlers +from ..services.background_task_periodic_heartbeat_check_setup import ( + setup as setup_background_task_periodic_heartbeat_check, +) +from ..services.modules.db import setup as setup_db +from ..services.modules.rabbitmq import setup as setup_rabbitmq +from ..services.modules.redis import setup as setup_redis +from ..services.modules.s3 import setup as setup_s3 +from ..services.process_message_running_service_setup import ( + setup as setup_process_message_running_service, +) from .settings import ApplicationSettings _logger = logging.getLogger(__name__) @@ -52,18 +51,6 @@ def create_app(settings: ApplicationSettings) -> FastAPI: # PLUGINS SETUP setup_api_routes(app) - if app.state.settings.RESOURCE_USAGE_TRACKER_PROMETHEUS_INSTRUMENTATION_ENABLED: - setup_prometheus_instrumentation(app) - if app.state.settings.RESOURCE_USAGE_TRACKER_TRACING: - setup_tracing( - app, - app.state.settings.RESOURCE_USAGE_TRACKER_TRACING, - app.state.settings.APP_NAME, - ) - - # ERROR HANDLERS - app.add_exception_handler(CustomResourceUsageTrackerError, http404_error_handler) - if settings.RESOURCE_USAGE_TRACKER_POSTGRES: setup_db(app) setup_redis(app) @@ -72,8 +59,20 @@ def create_app(settings: ApplicationSettings) -> FastAPI: # Needed for CSV export functionality setup_s3(app) - setup_resource_tracker(app) - setup_rpc_api_routes(app) + setup_rpc_api_routes(app) # Requires Rabbit, S3 + setup_background_task_periodic_heartbeat_check(app) # Requires Redis, DB + + setup_process_message_running_service(app) # Requires Rabbit + + if app.state.settings.RESOURCE_USAGE_TRACKER_TRACING: + setup_tracing( + app, + app.state.settings.RESOURCE_USAGE_TRACKER_TRACING, + app.state.settings.APP_NAME, + ) + + # ERROR HANDLERS + setup_exception_handlers(app) # EVENTS async def _on_startup() -> None: diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py deleted file mode 100644 index 298e63aef71..00000000000 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py +++ /dev/null @@ -1,20 +0,0 @@ -from fastapi import Request, status -from fastapi.responses import JSONResponse -from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( - CustomResourceUsageTrackerError, - ResourceUsageTrackerRuntimeError, -) - - -class ConfigurationError(ResourceUsageTrackerRuntimeError): - msg_template: str = "Application misconfiguration: {msg}" - - -def http404_error_handler( - request: Request, # pylint: disable=unused-argument - error: CustomResourceUsageTrackerError, -) -> JSONResponse: - return JSONResponse( - status_code=status.HTTP_404_NOT_FOUND, - content={"message": f"{error.msg_template}"}, - ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/__init__.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/__init__.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/__init__.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/__init__.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/errors.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/errors.py new file mode 100644 index 00000000000..533bec1b114 --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/errors.py @@ -0,0 +1,70 @@ +from models_library.errors_classes import OsparcErrorMixin + + +class ResourceUsageTrackerBaseError(OsparcErrorMixin, Exception): + msg_template = "Resource usage Tracker Service Error" + + +class ConfigurationError(ResourceUsageTrackerBaseError): + ... + + +### NotCreatedDBError + + +class NotCreatedDBError(ResourceUsageTrackerBaseError): + msg_template = "Data was not inserted to the DB. Data: {data}" + + +class ServiceRunNotCreatedDBError(NotCreatedDBError): + ... + + +class CreditTransactionNotCreatedDBError(NotCreatedDBError): + ... + + +class PricingPlanNotCreatedDBError(NotCreatedDBError): + ... + + +class PricingUnitNotCreatedDBError(NotCreatedDBError): + ... + + +class PricingUnitCostNotCreatedDBError(NotCreatedDBError): + ... + + +class PricingPlanToServiceNotCreatedDBError(NotCreatedDBError): + ... + + +### DoesNotExistsDBError + + +class PricingPlanDoesNotExistsDBError(ResourceUsageTrackerBaseError): + msg_template = "Pricing plan {pricing_plan_id} does not exists" + + +class PricingPlanAndPricingUnitCombinationDoesNotExistsDBError( + ResourceUsageTrackerBaseError +): + msg_template = "Pricing plan {pricing_plan_id} and pricing unit {pricing_unit_id} does not exists in product {product_name}" + + +class PricingUnitCostDoesNotExistsDBError(ResourceUsageTrackerBaseError): + msg_template = "Pricing unit cost id {pricing_unit_cost_id} does not exists" + + +### NotFoundError + + +class RutNotFoundError(ResourceUsageTrackerBaseError): + ... + + +class PricingPlanNotFoundForServiceError(RutNotFoundError): + msg_template = ( + "Pricing plan not found for service key {service_key} version {service_version}" + ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/__init__.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/__init__.py new file mode 100644 index 00000000000..8be40a236cf --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/__init__.py @@ -0,0 +1,20 @@ +from fastapi import FastAPI, HTTPException, status + +from ..errors import RutNotFoundError +from ._http_error import ( + http404_error_handler, + http_error_handler, + make_http_error_handler_for_exception, +) + + +def setup_exception_handlers(app: FastAPI) -> None: + app.add_exception_handler(HTTPException, http_error_handler) + app.add_exception_handler(RutNotFoundError, http404_error_handler) + + app.add_exception_handler( + Exception, + make_http_error_handler_for_exception( + status.HTTP_500_INTERNAL_SERVER_ERROR, Exception + ), + ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py new file mode 100644 index 00000000000..7879d27ae6f --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py @@ -0,0 +1,61 @@ +import logging +from collections.abc import Callable +from typing import Awaitable + +from fastapi import HTTPException, Request, status +from fastapi.encoders import jsonable_encoder +from servicelib.logging_errors import create_troubleshotting_log_kwargs +from servicelib.status_codes_utils import is_5xx_server_error +from starlette.responses import JSONResponse + +from ...exceptions.errors import RutNotFoundError + +_logger = logging.getLogger(__name__) + + +async def http_error_handler(request: Request, exc: Exception) -> JSONResponse: + assert isinstance(exc, HTTPException) # nosec + + if is_5xx_server_error(exc.status_code): + _logger.exception( + **create_troubleshotting_log_kwargs( + "Unexpected error happened in the Resource Usage Tracker. Please contact support.", + error=exc, + error_context={ + "request": request, + "request.method": f"{request.method}", + }, + ) + ) + return JSONResponse( + content=jsonable_encoder({"errors": [exc.detail]}), status_code=exc.status_code + ) + + +def http404_error_handler( + _: Request, # pylint: disable=unused-argument + exc: RutNotFoundError, +) -> JSONResponse: + return JSONResponse( + status_code=status.HTTP_404_NOT_FOUND, + content={"message": f"{exc.msg_template}"}, + ) + + +def make_http_error_handler_for_exception( + status_code: int, exception_cls: type[BaseException] +) -> Callable[[Request, type[BaseException]], Awaitable[JSONResponse]]: + """ + Produces a handler for BaseException-type exceptions which converts them + into an error JSON response with a given status code + + SEE https://docs.python.org/3/library/exceptions.html#concrete-exceptions + """ + + async def _http_error_handler(_: Request, exc: type[BaseException]) -> JSONResponse: + assert isinstance(exc, exception_cls) # nosec + return JSONResponse( + content=jsonable_encoder({"errors": [str(exc)]}), status_code=status_code + ) + + return _http_error_handler diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/web_main.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/main.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/web_main.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/main.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_credit_transactions.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/credit_transactions.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_credit_transactions.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/credit_transactions.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_plans.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/pricing_plans.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_plans.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/pricing_plans.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/pricing_unit_costs.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/pricing_unit_costs.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_units.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/pricing_units.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_units.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/pricing_units.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/service_runs.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/service_runs.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/prometheus.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/prometheus.py deleted file mode 100644 index 5421c1e8749..00000000000 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/prometheus.py +++ /dev/null @@ -1,73 +0,0 @@ -import logging -from typing import cast - -import requests -from fastapi import FastAPI -from prometheus_api_client import PrometheusConnect # type: ignore[import-untyped] -from servicelib.logging_utils import log_context -from settings_library.prometheus import PrometheusSettings -from tenacity import retry -from tenacity.before_sleep import before_sleep_log -from tenacity.retry import retry_if_exception_type -from tenacity.stop import stop_after_delay -from tenacity.wait import wait_random_exponential - -from ..core.errors import ConfigurationError - -_logger = logging.getLogger(__name__) - - -@retry( - reraise=True, - retry=retry_if_exception_type(ConfigurationError), - before_sleep=before_sleep_log(_logger, logging.WARNING), - stop=stop_after_delay(120), - wait=wait_random_exponential(max=30), -) -async def _wait_till_prometheus_responsive(client: PrometheusConnect) -> bool: - try: - connected: bool = client.check_prometheus_connection() - return connected - except requests.ConnectionError as exc: - raise ConfigurationError( - msg="Prometheus API client could not be reached. TIP: check configuration" - ) from exc - - -async def create_client(settings: PrometheusSettings) -> PrometheusConnect: - with log_context(_logger, logging.INFO, msg="connect with prometheus"): - client = PrometheusConnect(f"{settings.api_url}") - if await _wait_till_prometheus_responsive(client) is False: - raise ConfigurationError( - msg="Prometheus API client could be reached but returned value is not expected. TIP: check configuration" - ) - return client - - -def setup(app: FastAPI) -> None: - async def on_startup() -> None: - app.state.prometheus_api_client = None - settings: PrometheusSettings | None = ( - app.state.settings.RESOURCE_USAGE_TRACKER_PROMETHEUS - ) - if not settings: - _logger.warning("Prometheus API client is de-activated in the settings") - return - - app.state.prometheus_api_client = await create_client(settings) - - async def on_shutdown() -> None: - if app.state.prometheus_api_client: - with log_context(_logger, logging.INFO, msg="disconnect with prometheus"): - del app.state.prometheus_api_client - - app.add_event_handler("startup", on_startup) - app.add_event_handler("shutdown", on_shutdown) - - -def get_prometheus_api_client(app: FastAPI) -> PrometheusConnect: - if not app.state.prometheus_api_client: - raise ConfigurationError( - msg="Prometheus API client is not available. Please check the configuration." - ) - return cast(PrometheusConnect, app.state.prometheus_api_client) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_background_task.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check.py similarity index 94% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_background_task.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check.py index 213dd706582..256b737d479 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_background_task.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check.py @@ -11,13 +11,11 @@ ) from pydantic import NonNegativeInt, PositiveInt -from .core.settings import ApplicationSettings -from .models.resource_tracker_credit_transactions import ( - CreditTransactionCreditsAndStatusUpdate, -) -from .models.resource_tracker_service_runs import ServiceRunStoppedAtUpdate +from ..core.settings import ApplicationSettings +from ..models.credit_transactions import CreditTransactionCreditsAndStatusUpdate +from ..models.service_runs import ServiceRunStoppedAtUpdate from .modules.db.repositories.resource_tracker import ResourceTrackerRepository -from .resource_tracker_utils import compute_service_run_credit_costs, make_negative +from .utils import compute_service_run_credit_costs, make_negative _logger = logging.getLogger(__name__) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check_setup.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check_setup.py new file mode 100644 index 00000000000..9f022e863e5 --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check_setup.py @@ -0,0 +1,74 @@ +import logging +from collections.abc import Awaitable, Callable +from typing import TypedDict + +from fastapi import FastAPI +from servicelib.background_task import stop_periodic_task +from servicelib.logging_utils import log_catch, log_context +from servicelib.redis_utils import start_exclusive_periodic_task + +from ..core.settings import ApplicationSettings +from .background_task_periodic_heartbeat_check import ( + periodic_check_of_running_services_task, +) +from .modules.redis import get_redis_lock_client + +_logger = logging.getLogger(__name__) + + +_TASK_NAME_PERIODICALY_CHECK_RUNNING_SERVICES = "periodic_check_of_running_services" + + +class RutBackgroundTask(TypedDict): + name: str + task_func: Callable + + +def _on_app_startup(app: FastAPI) -> Callable[[], Awaitable[None]]: + async def _startup() -> None: + with log_context( + _logger, + logging.INFO, + msg="RUT background task Periodic check of running services startup..", + ), log_catch(_logger, reraise=False): + app_settings: ApplicationSettings = app.state.settings + + app.state.rut_background_task__periodic_check_of_running_services = None + + # Setup periodic task + exclusive_task = start_exclusive_periodic_task( + get_redis_lock_client(app), + periodic_check_of_running_services_task, + task_period=app_settings.RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC, + retry_after=app_settings.RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC, + task_name=_TASK_NAME_PERIODICALY_CHECK_RUNNING_SERVICES, + app=app, + ) + app.state.rut_background_task__periodic_check_of_running_services = ( + exclusive_task + ) + + return _startup + + +def _on_app_shutdown( + _app: FastAPI, +) -> Callable[[], Awaitable[None]]: + async def _stop() -> None: + with log_context( + _logger, + logging.INFO, + msg="RUT background tasks Periodic check of running services shutdown..", + ), log_catch(_logger, reraise=False): + assert _app # nosec + if _app.state.rut_background_task__periodic_check_of_running_services: + await stop_periodic_task( + _app.state.rut_background_task__periodic_check_of_running_services + ) + + return _stop + + +def setup(app: FastAPI) -> None: + app.add_event_handler("startup", _on_app_startup(app)) + app.add_event_handler("shutdown", _on_app_shutdown(app)) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_credit_transactions.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/credit_transactions.py similarity index 89% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_credit_transactions.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/credit_transactions.py index 8cd0bf9f647..0d4362e9748 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_credit_transactions.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/credit_transactions.py @@ -15,10 +15,10 @@ from servicelib.rabbitmq import RabbitMQClient from ..api.rest.dependencies import get_repository -from ..models.resource_tracker_credit_transactions import CreditTransactionCreate -from ..modules.db.repositories.resource_tracker import ResourceTrackerRepository -from ..modules.rabbitmq import get_rabbitmq_client_from_request -from ..resource_tracker_utils import sum_credit_transactions_and_publish_to_rabbitmq +from ..models.credit_transactions import CreditTransactionCreate +from .modules.db.repositories.resource_tracker import ResourceTrackerRepository +from .modules.rabbitmq import get_rabbitmq_client_from_request +from .utils import sum_credit_transactions_and_publish_to_rabbitmq async def create_credit_transaction( diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/__init__.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/__init__.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/__init__.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/__init__.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/__init__.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/__init__.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/__init__.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/__init__.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/__init__.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/_base.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/_base.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/_base.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/_base.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py similarity index 96% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py index 231c97502fb..33a3e58d137 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py @@ -28,9 +28,6 @@ from models_library.users import UserID from models_library.wallets import WalletID from pydantic import PositiveInt -from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( - CustomResourceUsageTrackerError, -) from simcore_postgres_database.models.resource_tracker_credit_transactions import ( resource_tracker_credit_transactions, ) @@ -51,19 +48,30 @@ ) from sqlalchemy.dialects.postgresql import ARRAY, INTEGER -from ....models.resource_tracker_credit_transactions import ( +from .....exceptions.errors import ( + CreditTransactionNotCreatedDBError, + PricingPlanAndPricingUnitCombinationDoesNotExistsDBError, + PricingPlanDoesNotExistsDBError, + PricingPlanNotCreatedDBError, + PricingPlanToServiceNotCreatedDBError, + PricingUnitCostDoesNotExistsDBError, + PricingUnitCostNotCreatedDBError, + PricingUnitNotCreatedDBError, + ServiceRunNotCreatedDBError, +) +from .....models.credit_transactions import ( CreditTransactionCreate, CreditTransactionCreditsAndStatusUpdate, CreditTransactionCreditsUpdate, ) -from ....models.resource_tracker_pricing_plans import ( +from .....models.pricing_plans import ( PricingPlansDB, PricingPlansWithServiceDefaultPlanDB, PricingPlanToServiceDB, ) -from ....models.resource_tracker_pricing_unit_costs import PricingUnitCostsDB -from ....models.resource_tracker_pricing_units import PricingUnitsDB -from ....models.resource_tracker_service_runs import ( +from .....models.pricing_unit_costs import PricingUnitCostsDB +from .....models.pricing_units import PricingUnitsDB +from .....models.service_runs import ( OsparcCreditsAggregatedByServiceKeyDB, ServiceRunCreate, ServiceRunDB, @@ -125,9 +133,7 @@ async def create_service_run(self, data: ServiceRunCreate) -> ServiceRunId: result = await conn.execute(insert_stmt) row = result.first() if row is None: - raise CustomResourceUsageTrackerError( - msg=f"Service was not created: {data}" - ) + raise ServiceRunNotCreatedDBError(data=data) return cast(ServiceRunId, row[0]) async def update_service_run_last_heartbeat( @@ -655,9 +661,7 @@ async def create_credit_transaction( result = await conn.execute(insert_stmt) row = result.first() if row is None: - raise CustomResourceUsageTrackerError( - msg=f"Transaction was not created: {data}" - ) + raise CreditTransactionNotCreatedDBError(data=data) return cast(CreditTransactionId, row[0]) async def update_credit_transaction_credits( @@ -870,9 +874,7 @@ async def get_pricing_plan( result = await conn.execute(select_stmt) row = result.first() if row is None: - raise CustomResourceUsageTrackerError( - msg=f"Pricing plan does not exists: {pricing_plan_id}" - ) + raise PricingPlanDoesNotExistsDBError(pricing_plan_id=pricing_plan_id) return PricingPlansDB.from_orm(row) async def list_pricing_plans_by_product( @@ -921,9 +923,7 @@ async def create_pricing_plan(self, data: PricingPlanCreate) -> PricingPlansDB: result = await conn.execute(insert_stmt) row = result.first() if row is None: - raise CustomResourceUsageTrackerError( - msg=f"Pricing plan was not created: {data}" - ) + raise PricingPlanNotCreatedDBError(data=data) return PricingPlansDB.from_orm(row) async def update_pricing_plan( @@ -1084,8 +1084,8 @@ async def upsert_service_to_pricing_plan( result = await conn.execute(insert_stmt) row = result.first() if row is None: - raise CustomResourceUsageTrackerError( - msg="Pricing plan to service record was not created" + raise PricingPlanToServiceNotCreatedDBError( + data=f"pricing_plan_id {pricing_plan_id}, service_key {service_key}, service_version {service_version}" ) return PricingPlanToServiceDB.from_orm(row) @@ -1194,8 +1194,10 @@ async def get_valid_pricing_unit( row = result.first() if row is None: - raise CustomResourceUsageTrackerError( - msg=f"Pricing plan {pricing_plan_id} and pricing unit {pricing_unit_id} for product {product_name} not found" + raise PricingPlanAndPricingUnitCombinationDoesNotExistsDBError( + pricing_plan_id=pricing_plan_id, + pricing_unit_id=pricing_unit_id, + product_name=product_name, ) return PricingUnitsDB.from_orm(row) @@ -1220,9 +1222,7 @@ async def create_pricing_unit_with_cost( result = await conn.execute(insert_stmt) row = result.first() if row is None: - raise CustomResourceUsageTrackerError( - msg=f"Pricing unit was not created: {data}" - ) + raise PricingUnitNotCreatedDBError(data=data) _pricing_unit_id = row[0] # pricing unit cost table @@ -1245,9 +1245,7 @@ async def create_pricing_unit_with_cost( result = await conn.execute(insert_stmt) row = result.first() if row is None: - raise CustomResourceUsageTrackerError( - msg=f"Pricing unit cost was not created: {data}" - ) + raise PricingUnitCostNotCreatedDBError(data=data) _pricing_unit_cost_id = row[0] return (_pricing_unit_id, _pricing_unit_cost_id) @@ -1313,9 +1311,7 @@ async def update_pricing_unit_with_cost( result = await conn.execute(insert_stmt) row = result.first() if row is None: - raise CustomResourceUsageTrackerError( - msg=f"Pricing unit cost was not created: {data}" - ) + raise PricingUnitCostNotCreatedDBError(data=data) ################################# # Pricing unit-costs @@ -1345,7 +1341,7 @@ async def get_pricing_unit_cost_by_id( row = result.first() if row is None: - raise CustomResourceUsageTrackerError( - msg=f"Pricing unit cost id {pricing_unit_cost_id} not found in the resource_tracker_pricing_unit_costs table", + raise PricingUnitCostDoesNotExistsDBError( + pricing_unit_cost_id=pricing_unit_cost_id ) return PricingUnitCostsDB.from_orm(row) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/rabbitmq.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/rabbitmq.py similarity index 97% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/rabbitmq.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/rabbitmq.py index 9bb4ec3d2b7..57fb01bdcbf 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/rabbitmq.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/rabbitmq.py @@ -10,7 +10,7 @@ ) from settings_library.rabbit import RabbitSettings -from ..core.errors import ConfigurationError +from ...exceptions.errors import ConfigurationError logger = logging.getLogger(__name__) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/redis.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/redis.py similarity index 94% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/redis.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/redis.py index 354f2fea5d2..922b0e7e49e 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/redis.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/redis.py @@ -25,5 +25,5 @@ async def on_shutdown() -> None: app.add_event_handler("shutdown", on_shutdown) -def get_redis_client(app: FastAPI) -> RedisClientSDK: +def get_redis_lock_client(app: FastAPI) -> RedisClientSDK: return cast(RedisClientSDK, app.state.redis_client_sdk) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/s3.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/s3.py similarity index 97% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/s3.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/s3.py index 889b8cfcd1c..cc41206c256 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/s3.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/s3.py @@ -12,7 +12,7 @@ wait_random_exponential, ) -from ..core.errors import ConfigurationError +from ...exceptions.errors import ConfigurationError _logger = logging.getLogger(__name__) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_plans.py similarity index 93% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_plans.py index d37f244dbc9..e597806d98c 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_plans.py @@ -13,17 +13,12 @@ PricingPlanUpdate, ) from models_library.services import ServiceKey, ServiceVersion -from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( - CustomResourceUsageTrackerError, -) from ..api.rest.dependencies import get_repository -from ..models.resource_tracker_pricing_plans import ( - PricingPlansDB, - PricingPlanToServiceDB, -) -from ..models.resource_tracker_pricing_units import PricingUnitsDB -from ..modules.db.repositories.resource_tracker import ResourceTrackerRepository +from ..exceptions.errors import PricingPlanNotFoundForServiceError +from ..models.pricing_plans import PricingPlansDB, PricingPlanToServiceDB +from ..models.pricing_units import PricingUnitsDB +from .modules.db.repositories.resource_tracker import ResourceTrackerRepository async def _create_pricing_plan_get( @@ -71,8 +66,8 @@ async def get_service_default_pricing_plan( break if default_pricing_plan is None: - raise CustomResourceUsageTrackerError( - msg="No default pricing plan for the specified service" + raise PricingPlanNotFoundForServiceError( + service_key=service_key, service_version=service_version ) pricing_plan_unit_db = ( diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_units.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_units.py similarity index 97% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_units.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_units.py index 0194cc0f699..f2aee53dd80 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_units.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_units.py @@ -13,7 +13,7 @@ ) from ..api.rest.dependencies import get_repository -from ..modules.db.repositories.resource_tracker import ResourceTrackerRepository +from .modules.db.repositories.resource_tracker import ResourceTrackerRepository async def get_pricing_unit( diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/process_message_running_service.py similarity index 98% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/process_message_running_service.py index 4cb024b3b7c..4352e327266 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/process_message_running_service.py @@ -22,19 +22,19 @@ from models_library.services import ServiceType from pydantic import parse_raw_as -from .models.resource_tracker_credit_transactions import ( +from ..models.credit_transactions import ( CreditTransactionCreate, CreditTransactionCreditsAndStatusUpdate, CreditTransactionCreditsUpdate, ) -from .models.resource_tracker_service_runs import ( +from ..models.service_runs import ( ServiceRunCreate, ServiceRunLastHeartbeatUpdate, ServiceRunStoppedAtUpdate, ) from .modules.db.repositories.resource_tracker import ResourceTrackerRepository from .modules.rabbitmq import RabbitMQClient, get_rabbitmq_client -from .resource_tracker_utils import ( +from .utils import ( compute_service_run_credit_costs, make_negative, publish_to_rabbitmq_wallet_credits_limit_reached, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/process_message_running_service_setup.py similarity index 61% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/process_message_running_service_setup.py index 6bee4c87add..3624eb1254d 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/process_message_running_service_setup.py @@ -4,22 +4,16 @@ from fastapi import FastAPI from models_library.rabbitmq_messages import RabbitResourceTrackingBaseMessage -from servicelib.background_task import stop_periodic_task from servicelib.logging_utils import log_catch, log_context from servicelib.rabbitmq import RabbitMQClient -from servicelib.redis_utils import start_exclusive_periodic_task from settings_library.rabbit import RabbitSettings -from .core.settings import ApplicationSettings +from ..core.settings import ApplicationSettings from .modules.rabbitmq import get_rabbitmq_client -from .modules.redis import get_redis_client -from .resource_tracker_background_task import periodic_check_of_running_services_task -from .resource_tracker_process_messages import process_message +from .process_message_running_service import process_message _logger = logging.getLogger(__name__) -_TASK_NAME_PERIODICALY_CHECK_RUNNING_SERVICES = "periodic_check_of_running_services" - _RUT_MESSAGE_TTL_IN_MS = 2 * 60 * 60 * 1000 # 2 hours @@ -42,7 +36,6 @@ async def _startup() -> None: ), log_catch(_logger, reraise=False): app_settings: ApplicationSettings = app.state.settings app.state.resource_tracker_rabbitmq_consumer = None - app.state.resource_tracker_background_task = None settings: RabbitSettings | None = ( app_settings.RESOURCE_USAGE_TRACKER_RABBITMQ ) @@ -52,16 +45,6 @@ async def _startup() -> None: app.state.resource_tracker_rabbitmq_consumer = await _subscribe_to_rabbitmq( app ) - # Setup periodic task that will try to run "periodic_check_of_running_services_task" - if app_settings.RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED: - app.state.resource_tracker_background_task = start_exclusive_periodic_task( - get_redis_client(app), - periodic_check_of_running_services_task, - task_period=app_settings.RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC, - retry_after=app_settings.RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC, - task_name=_TASK_NAME_PERIODICALY_CHECK_RUNNING_SERVICES, - app=app, - ) return _startup @@ -72,8 +55,6 @@ def on_app_shutdown( async def _stop() -> None: # NOTE: We want to have persistent queue, therefore we will not unsubscribe assert _app # nosec - if _app.state.resource_tracker_background_task: - await stop_periodic_task(_app.state.resource_tracker_background_task) return _stop diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py similarity index 96% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py index 3f2167e3d77..782b084c789 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py @@ -19,12 +19,9 @@ from models_library.users import UserID from models_library.wallets import WalletID from pydantic import AnyUrl, PositiveInt -from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( - CustomResourceUsageTrackerError, -) -from ..models.resource_tracker_service_runs import ServiceRunWithCreditsDB -from ..modules.db.repositories.resource_tracker import ResourceTrackerRepository +from ..models.service_runs import ServiceRunWithCreditsDB +from .modules.db.repositories.resource_tracker import ResourceTrackerRepository _PRESIGNED_LINK_EXPIRATION_SEC = 7200 @@ -111,7 +108,7 @@ async def list_service_runs( ) else: msg = "wallet_id and access_all_wallet_usage parameters must be specified together" - raise CustomResourceUsageTrackerError(msg=msg) + raise ValueError(msg) service_runs_api_model: list[ServiceRunGet] = [] for service in service_runs_db_model: diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/utils.py similarity index 100% rename from services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py rename to services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/utils.py diff --git a/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py b/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py index cffb606fae5..66052119d72 100644 --- a/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py +++ b/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py @@ -27,7 +27,7 @@ def test_healthcheck( rabbitmq_mock = mocker.Mock(spec=RabbitMQClient) rabbitmq_mock.healthy = True mocker.patch( - "simcore_service_resource_usage_tracker.modules.rabbitmq.get_rabbitmq_client", + "simcore_service_resource_usage_tracker.services.modules.rabbitmq.get_rabbitmq_client", return_value=rabbitmq_mock, ) @@ -50,7 +50,7 @@ def test_healthcheck__unhealthy( rabbitmq_mock = mocker.Mock(spec=RabbitMQClient) rabbitmq_mock.healthy = False mocker.patch( - "simcore_service_resource_usage_tracker.modules.rabbitmq.get_rabbitmq_client", + "simcore_service_resource_usage_tracker.services.modules.rabbitmq.get_rabbitmq_client", return_value=rabbitmq_mock, ) diff --git a/services/resource-usage-tracker/tests/unit/conftest.py b/services/resource-usage-tracker/tests/unit/conftest.py index 6d70142d5e4..6e4bfcd6b98 100644 --- a/services/resource-usage-tracker/tests/unit/conftest.py +++ b/services/resource-usage-tracker/tests/unit/conftest.py @@ -68,6 +68,7 @@ def app_environment( "PROMETHEUS_USERNAME": faker.user_name(), "PROMETHEUS_PASSWORD": faker.password(special_chars=False), "RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED": "0", + "RESOURCE_USAGE_TRACKER_TRACING": "null", }, ) diff --git a/services/resource-usage-tracker/tests/unit/modules/test_rabbitmq.py b/services/resource-usage-tracker/tests/unit/modules/test_rabbitmq.py index dde606241e4..f69d8989b24 100644 --- a/services/resource-usage-tracker/tests/unit/modules/test_rabbitmq.py +++ b/services/resource-usage-tracker/tests/unit/modules/test_rabbitmq.py @@ -5,7 +5,9 @@ from fastapi import FastAPI from settings_library.rabbit import RabbitSettings -from simcore_service_resource_usage_tracker.modules.rabbitmq import get_rabbitmq_client +from simcore_service_resource_usage_tracker.services.modules.rabbitmq import ( + get_rabbitmq_client, +) # Selection of core and tool services started in this swarm fixture (integration) pytest_simcore_core_services_selection = [ diff --git a/services/resource-usage-tracker/tests/unit/modules/test_redis.py b/services/resource-usage-tracker/tests/unit/modules/test_redis.py index 1a7d79b202d..e518c589c79 100644 --- a/services/resource-usage-tracker/tests/unit/modules/test_redis.py +++ b/services/resource-usage-tracker/tests/unit/modules/test_redis.py @@ -6,7 +6,9 @@ from unittest import mock from fastapi import FastAPI -from simcore_service_resource_usage_tracker.modules.redis import get_redis_client +from simcore_service_resource_usage_tracker.services.modules.redis import ( + get_redis_lock_client, +) async def test_redis_raises_if_missing( @@ -17,5 +19,5 @@ async def test_redis_raises_if_missing( mocked_redis_server: None, initialized_app: FastAPI, ): - client = get_redis_client(initialized_app) + client = get_redis_lock_client(initialized_app) assert await client.ping() is True diff --git a/services/resource-usage-tracker/tests/unit/test_computation_of_credits.py b/services/resource-usage-tracker/tests/unit/test_computation_of_credits.py index 5264b799f33..bd48460457c 100644 --- a/services/resource-usage-tracker/tests/unit/test_computation_of_credits.py +++ b/services/resource-usage-tracker/tests/unit/test_computation_of_credits.py @@ -7,7 +7,7 @@ from decimal import Decimal import pytest -from simcore_service_resource_usage_tracker.resource_tracker_utils import ( +from simcore_service_resource_usage_tracker.services.utils import ( compute_service_run_credit_costs, ) diff --git a/services/resource-usage-tracker/tests/unit/test_web_main.py b/services/resource-usage-tracker/tests/unit/test_main.py similarity index 75% rename from services/resource-usage-tracker/tests/unit/test_web_main.py rename to services/resource-usage-tracker/tests/unit/test_main.py index a58f4a6139a..6d9addd8ee2 100644 --- a/services/resource-usage-tracker/tests/unit/test_web_main.py +++ b/services/resource-usage-tracker/tests/unit/test_main.py @@ -7,6 +7,6 @@ def test_main_app(app_environment: EnvVarsDict): - from simcore_service_resource_usage_tracker.web_main import the_app, the_settings + from simcore_service_resource_usage_tracker.main import the_app, the_settings assert the_app.state.settings == the_settings diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py index 6f240d658ee..581952e1100 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py @@ -31,12 +31,10 @@ ) from simcore_service_resource_usage_tracker.core.application import create_app from simcore_service_resource_usage_tracker.core.settings import ApplicationSettings -from simcore_service_resource_usage_tracker.models.resource_tracker_credit_transactions import ( +from simcore_service_resource_usage_tracker.models.credit_transactions import ( CreditTransactionDB, ) -from simcore_service_resource_usage_tracker.models.resource_tracker_service_runs import ( - ServiceRunDB, -) +from simcore_service_resource_usage_tracker.models.service_runs import ServiceRunDB from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/data/list_of_prometheus_mocked_outputs.json b/services/resource-usage-tracker/tests/unit/with_dbs/data/list_of_prometheus_mocked_outputs.json deleted file mode 100644 index eec1a5cca04..00000000000 --- a/services/resource-usage-tracker/tests/unit/with_dbs/data/list_of_prometheus_mocked_outputs.json +++ /dev/null @@ -1,496 +0,0 @@ -[ - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmpb1ztyf6m", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/ec08331c8658286a384cfa6e57f3b4daa59c9e9a369ed42401f621c142042f40", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-11177106-2302-5461-9017-e495145858b6-0-jupyter-smash" - }, - "values": [ - [ - 1686907140, - "2.5380915280000007" - ], - [ - 1686907200, - "18.057210595000004" - ], - [ - 1686907260, - "18.057210595000004" - ], - [ - 1686907320, - "18.057210595000004" - ], - [ - 1686907380, - "18.057210595000004" - ], - [ - 1686907440, - "18.057210595000004" - ] - ] - }, - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmpqg_rpq7e", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/2873c8bee46543f059d4c2c2500ba658d24bdf0abdfcdc2b7eccc955e9538c43", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-9069c179-c3d7-58a6-8431-8cb19339c847-0-jupyter-smash" - }, - "values": [ - [ - 1686907080, - "2.5288082430000003" - ], - [ - 1686907140, - "16.675726328" - ], - [ - 1686907200, - "17.503259114" - ], - [ - 1686907260, - "17.503259114" - ], - [ - 1686907320, - "17.503259114" - ], - [ - 1686907380, - "17.503259114" - ], - [ - 1686907440, - "17.503259114" - ] - ] - }, - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmp0m7ijyoy", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/b6cd6d013763a34795aaac8402b7f8943df80c2f63ba4f90f95c5b7235dbbc4e", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-8f1a908e-8b4b-5011-a42c-98abd141fc50-0-jupyter-smash" - }, - "values": [ - [ - 1686907140, - "5.765171485000001" - ], - [ - 1686907200, - "5.835658748999999" - ], - [ - 1686907260, - "5.835658748999999" - ], - [ - 1686907320, - "5.835658748999999" - ], - [ - 1686907380, - "5.835658748999999" - ], - [ - 1686907440, - "5.835658748999999" - ] - ] - }, - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmp3o6q4v5e", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/22394ffafb00907311fc6b76423875411f1084009f89fe77bc618f48f06053eb", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-14387f03-4a37-5187-b8e0-0e491389ade4-0-jupyter-smash" - }, - "values": [ - [ - 1686907200, - "2.500054865" - ], - [ - 1686907260, - "7.3528712789999995" - ], - [ - 1686907320, - "17.330580469999997" - ], - [ - 1686907380, - "17.330580469999997" - ], - [ - 1686907440, - "17.330580469999997" - ], - [ - 1686907500, - "17.330580469999997" - ], - [ - 1686907560, - "17.330580469999997" - ] - ] - }, - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmpfgfyp7n5", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/fecc93800827e3fbdaee9a51842ff2001d010f809c46e806f74611bada02b7b1", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-32f3c80d-7cbc-58d3-840e-fedae5252dd0-0-jupyter-smash" - }, - "values": [ - [ - 1686906660, - "4.4991374429999995" - ], - [ - 1686906720, - "5.768837835" - ], - [ - 1686906780, - "5.815829203" - ], - [ - 1686906840, - "5.815829203" - ], - [ - 1686906900, - "5.815829203" - ], - [ - 1686906960, - "5.815829203" - ], - [ - 1686907020, - "5.815829203" - ] - ] - }, - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmpyugvc3z8", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/38179a390b273788cba9a7b33860f0606c4d4c45d13d63f9d1920509e05d2054", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-f7c6e801-d8a0-5f57-8283-850adf128613-0-jupyter-smash" - }, - "values": [ - [ - 1686907080, - "5.086436862999999" - ], - [ - 1686907140, - "5.778702072999999" - ], - [ - 1686907200, - "5.795705571" - ], - [ - 1686907260, - "5.795705571" - ], - [ - 1686907320, - "5.795705571" - ], - [ - 1686907380, - "5.795705571" - ], - [ - 1686907440, - "5.795705571" - ] - ] - }, - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmpehms8dgc", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/98f90bfd2e28861247f550da8dcf09ebcfa3e5304d15f470708f67f84e724a8f", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-d283967b-11db-5d91-a484-af570f32d5e7-0-jupyter-smash" - }, - "values": [ - [ - 1686907440, - "5.755467759" - ], - [ - 1686907500, - "5.801798311000001" - ], - [ - 1686907560, - "5.801798311000001" - ] - ] - }, - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmpdkm5klhn", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/e468fa476815d843e3d3dcf4c7988c8846107be02959a2ef1fd41584b18c18ee", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-fee6dbfe-529f-5c57-8121-af8a507326f0-0-jupyter-smash" - }, - "values": [ - [ - 1686907200, - "2.623185233" - ], - [ - 1686907260, - "5.775293023000001" - ], - [ - 1686907320, - "5.818879645000001" - ], - [ - 1686907380, - "5.818879645000001" - ], - [ - 1686907440, - "5.818879645000001" - ], - [ - 1686907500, - "5.818879645000001" - ], - [ - 1686907560, - "5.818879645000001" - ] - ] - }, - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmp6y2a68v3", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/7f7d260357ee8c80f69b148bc090068df4bc1c9aafad909dc1557934d0366967", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-88ebaebf-c50e-5b33-a3ce-fe7ed2ad9668-0-jupyter-smash" - }, - "values": [ - [ - 1686906660, - "2.4900833509999996" - ], - [ - 1686906720, - "10.969049619000002" - ], - [ - 1686906780, - "17.436269327999998" - ], - [ - 1686906840, - "17.436269327999998" - ], - [ - 1686906900, - "17.436269327999998" - ], - [ - 1686906960, - "17.436269327999998" - ], - [ - 1686907020, - "17.436269327999998" - ] - ] - }, - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmp_3seh6kp", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/58e1138d51eb5eafd737024d0df0b01ef88f2087e5a3922565c59130d57ac7a3", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-2b231c38-0ebc-5cc0-9030-1ffe573f54e9-0-jupyter-smash" - }, - "values": [ - [ - 1686907440, - "5.157543565" - ], - [ - 1686907500, - "17.386926716" - ], - [ - 1686907560, - "17.386926716" - ] - ] - }, - { - "metric": { - "container_label_com_docker_compose_oneoff": "False", - "container_label_com_docker_compose_project_working_dir": "/tmp/tmp_3seh6kp", - "container_label_com_docker_compose_version": "1.29.1", - "container_label_io_simcore_runtime_product_name": "osparc", - "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", - "container_label_io_simcore_runtime_simcore_user_agent": "puppeteer", - "container_label_io_simcore_runtime_project_id": "df604058-0c26-11ee-994f-02420a0b0fc8", - "container_label_io_simcore_runtime_user_id": "43819", - "container_label_io_simcore_runtime_node_id": "14387f03-4a37-5187-b8e0-0e491389ade4", - "container_label_io_simcore_runtime_cpu_limit": "3.5", - "container_label_io_simcore_runtime_memory_limit": "17179869184", - "id": "/docker/58e1138d51eb5eafd737024d0df0b01ef88f2087e5a3922565c59130d57ac7a3", - "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", - "instance": "gpu1", - "job": "cadvisor", - "name": "dy-sidecar-2b231c38-0ebc-5cc0-9030-1ffe573f54e9-0-jupyter-smash" - }, - "values": [ - [ - 1686907440, - "5.157543565" - ], - [ - 1686907500, - "17.386926716" - ], - [ - 1686908560, - "20.846512345" - ] - ] - } -] diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py index c53c1accb90..37ea1fa8ac4 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py @@ -26,7 +26,7 @@ @pytest.fixture async def mocked_export(mocker: MockerFixture): mock_export = mocker.patch( - "simcore_service_resource_usage_tracker.services.resource_tracker_service_runs.ResourceTrackerRepository.export_service_runs_table_to_s3", + "simcore_service_resource_usage_tracker.services.service_runs.ResourceTrackerRepository.export_service_runs_table_to_s3", autospec=True, ) @@ -36,7 +36,7 @@ async def mocked_export(mocker: MockerFixture): @pytest.fixture async def mocked_presigned_link(mocker: MockerFixture): mock_presigned_link = mocker.patch( - "simcore_service_resource_usage_tracker.services.resource_tracker_service_runs.SimcoreS3API.create_single_presigned_download_link", + "simcore_service_resource_usage_tracker.services.service_runs.SimcoreS3API.create_single_presigned_download_link", return_value=parse_obj_as( AnyUrl, "https://www.testing.com/", diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__list_billable.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__list_billable.py index c9bd9e7a0c6..a9027414d5d 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__list_billable.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__list_billable.py @@ -13,10 +13,8 @@ ) from models_library.rest_ordering import OrderBy, OrderDirection from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq._errors import RPCServerError from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker import service_runs -from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( - CustomResourceUsageTrackerError, -) from simcore_postgres_database.models.resource_tracker_credit_transactions import ( resource_tracker_credit_transactions, ) @@ -188,10 +186,11 @@ async def test_rpc_list_service_runs_raising_custom_error( resource_tracker_setup_db: dict, rpc_client: RabbitMQRPCClient, ): - with pytest.raises(CustomResourceUsageTrackerError): + with pytest.raises(RPCServerError) as e: await service_runs.get_service_run_page( rpc_client, user_id=_USER_ID, product_name="osparc", access_all_wallet_usage=True, ) + assert e diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task_periodic_heartbeat_check.py similarity index 95% rename from services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py rename to services/resource-usage-tracker/tests/unit/with_dbs/test_background_task_periodic_heartbeat_check.py index 2b719326bc9..ce2f3f8a6db 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task_periodic_heartbeat_check.py @@ -16,18 +16,16 @@ resource_tracker_service_runs, ) from simcore_service_resource_usage_tracker.core.settings import ApplicationSettings -from simcore_service_resource_usage_tracker.models.resource_tracker_credit_transactions import ( +from simcore_service_resource_usage_tracker.models.credit_transactions import ( CreditTransactionDB, ) -from simcore_service_resource_usage_tracker.models.resource_tracker_service_runs import ( - ServiceRunDB, +from simcore_service_resource_usage_tracker.models.service_runs import ServiceRunDB +from simcore_service_resource_usage_tracker.services.background_task_periodic_heartbeat_check import ( + periodic_check_of_running_services_task, ) -from simcore_service_resource_usage_tracker.modules.db.repositories.resource_tracker import ( +from simcore_service_resource_usage_tracker.services.modules.db.repositories.resource_tracker import ( ResourceTrackerRepository, ) -from simcore_service_resource_usage_tracker.resource_tracker_background_task import ( - periodic_check_of_running_services_task, -) pytest_simcore_core_services_selection = ["postgres", "rabbit"] pytest_simcore_ops_services_selection = [ diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py index dec714a582b..da321f593f3 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py @@ -8,10 +8,10 @@ SimcorePlatformStatus, ) from servicelib.rabbitmq import RabbitMQClient -from simcore_service_resource_usage_tracker.modules.db.repositories.resource_tracker import ( +from simcore_service_resource_usage_tracker.services.modules.db.repositories.resource_tracker import ( ResourceTrackerRepository, ) -from simcore_service_resource_usage_tracker.resource_tracker_process_messages import ( +from simcore_service_resource_usage_tracker.services.process_message_running_service import ( _process_heartbeat_event, _process_start_event, _process_stop_event, diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py index 4b6c1a0dfac..4537d1fb6d2 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py @@ -31,10 +31,10 @@ resource_tracker_pricing_units, ) from simcore_postgres_database.models.services import services_meta_data -from simcore_service_resource_usage_tracker.modules.db.repositories.resource_tracker import ( +from simcore_service_resource_usage_tracker.services.modules.db.repositories.resource_tracker import ( ResourceTrackerRepository, ) -from simcore_service_resource_usage_tracker.resource_tracker_process_messages import ( +from simcore_service_resource_usage_tracker.services.process_message_running_service import ( _process_heartbeat_event, _process_start_event, _process_stop_event, diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py index c1d62af5b23..a3e69edac99 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py @@ -31,10 +31,10 @@ resource_tracker_pricing_units, ) from simcore_postgres_database.models.services import services_meta_data -from simcore_service_resource_usage_tracker.modules.db.repositories.resource_tracker import ( +from simcore_service_resource_usage_tracker.services.modules.db.repositories.resource_tracker import ( ResourceTrackerRepository, ) -from simcore_service_resource_usage_tracker.resource_tracker_process_messages import ( +from simcore_service_resource_usage_tracker.services.process_message_running_service import ( _process_heartbeat_event, _process_start_event, _process_stop_event, diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py index b71317b1aab..d8b4749a37a 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py @@ -26,9 +26,7 @@ parse_request_path_parameters_as, ) from servicelib.aiohttp.typing_extension import Handler -from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( - CustomResourceUsageTrackerError, -) +from servicelib.rabbitmq._errors import RPCServerError from servicelib.request_keys import RQT_USERID_KEY from .._constants import RQ_PRODUCT_KEY @@ -49,8 +47,10 @@ async def wrapper(request: web.Request) -> web.StreamResponse: try: return await handler(request) - except CustomResourceUsageTrackerError as exc: - raise CustomResourceUsageTrackerError from exc + except RPCServerError as exc: + # NOTE: This will be improved; we will add a mapping between + # RPC errors and user-friendly frontend errors to pass to the frontend. + raise RPCServerError from exc return wrapper From cba3237dafbc78f0513af5b32e5835bcf025553b Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 5 Nov 2024 08:00:46 +0100 Subject: [PATCH 05/22] =?UTF-8?q?=E2=99=BB=EF=B8=8FDirector-v2:=20refactor?= =?UTF-8?q?=20computational=20scheduler=20(#6652)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/servicelib/background_task.py | 25 +++++- .../tests/test_background_task.py | 78 +++++++++++++++---- .../api/dependencies/scheduler.py | 2 +- .../api/routes/computations.py | 4 +- .../core/application.py | 2 +- .../modules/comp_scheduler/__init__.py | 16 +++- .../{base_scheduler.py => _base_scheduler.py} | 0 .../{dask_scheduler.py => _dask_scheduler.py} | 18 +++-- .../{factory.py => _scheduler_factory.py} | 16 ++-- .../modules/comp_scheduler/_task.py | 50 ++++++++++++ .../modules/comp_scheduler/background_task.py | 73 ----------------- .../integration/01/test_computation_api.py | 2 + ...t_dynamic_sidecar_nodeports_integration.py | 1 + services/director-v2/tests/unit/_helpers.py | 4 +- .../with_dbs/test_api_route_computations.py | 4 +- ...t_modules_comp_scheduler_dask_scheduler.py | 37 +++++---- services/director/requirements/_tools.in | 7 -- services/director/requirements/_tools.txt | 29 ++++++- 18 files changed, 231 insertions(+), 137 deletions(-) rename services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/{base_scheduler.py => _base_scheduler.py} (100%) rename services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/{dask_scheduler.py => _dask_scheduler.py} (96%) rename services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/{factory.py => _scheduler_factory.py} (79%) create mode 100644 services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_task.py delete mode 100644 services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/background_task.py diff --git a/packages/service-library/src/servicelib/background_task.py b/packages/service-library/src/servicelib/background_task.py index e7a4c665c49..5bd2f09ecb6 100644 --- a/packages/service-library/src/servicelib/background_task.py +++ b/packages/service-library/src/servicelib/background_task.py @@ -25,15 +25,36 @@ class PeriodicTaskCancellationError(PydanticErrorMixin, Exception): msg_template: str = "Could not cancel task '{task_name}'" +class SleepUsingAsyncioEvent: + """Sleep strategy that waits on an event to be set.""" + + def __init__(self, event: "asyncio.Event") -> None: + self.event = event + + async def __call__(self, timeout: float | None) -> None: + with contextlib.suppress(TimeoutError): + await asyncio.wait_for(self.event.wait(), timeout=timeout) + self.event.clear() + + async def _periodic_scheduled_task( task: Callable[..., Awaitable[None]], *, interval: datetime.timedelta, task_name: str, + early_wake_up_event: asyncio.Event | None, **task_kwargs, ) -> None: # NOTE: This retries forever unless cancelled - async for attempt in AsyncRetrying(wait=wait_fixed(interval.total_seconds())): + nap = ( + asyncio.sleep + if early_wake_up_event is None + else SleepUsingAsyncioEvent(early_wake_up_event) + ) + async for attempt in AsyncRetrying( + sleep=nap, + wait=wait_fixed(interval.total_seconds()), + ): with attempt: with log_context( _logger, @@ -51,6 +72,7 @@ def start_periodic_task( interval: datetime.timedelta, task_name: str, wait_before_running: datetime.timedelta = datetime.timedelta(0), + early_wake_up_event: asyncio.Event | None = None, **kwargs, ) -> asyncio.Task: with log_context( @@ -64,6 +86,7 @@ def start_periodic_task( task, interval=interval, task_name=task_name, + early_wake_up_event=early_wake_up_event, **kwargs, ), name=task_name, diff --git a/packages/service-library/tests/test_background_task.py b/packages/service-library/tests/test_background_task.py index 59b2aaeff3f..8d69b5f5e2b 100644 --- a/packages/service-library/tests/test_background_task.py +++ b/packages/service-library/tests/test_background_task.py @@ -6,7 +6,8 @@ import asyncio import datetime -from typing import AsyncIterator, Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable +from typing import Final from unittest import mock import pytest @@ -18,13 +19,13 @@ stop_periodic_task, ) -_FAST_POLL_INTERVAL = 1 +_FAST_POLL_INTERVAL: Final[int] = 1 +_VERY_SLOW_POLL_INTERVAL: Final[int] = 100 @pytest.fixture def mock_background_task(mocker: MockerFixture) -> mock.AsyncMock: - mocked_task = mocker.AsyncMock(return_value=None) - return mocked_task + return mocker.AsyncMock(return_value=None) @pytest.fixture @@ -32,7 +33,12 @@ def task_interval() -> datetime.timedelta: return datetime.timedelta(seconds=_FAST_POLL_INTERVAL) -@pytest.fixture(params=[None, 1]) +@pytest.fixture +def very_long_task_interval() -> datetime.timedelta: + return datetime.timedelta(seconds=_VERY_SLOW_POLL_INTERVAL) + + +@pytest.fixture(params=[None, 1], ids=lambda x: f"stop-timeout={x}") def stop_task_timeout(request: pytest.FixtureRequest) -> float | None: return request.param @@ -40,16 +46,23 @@ def stop_task_timeout(request: pytest.FixtureRequest) -> float | None: @pytest.fixture async def create_background_task( faker: Faker, stop_task_timeout: float | None -) -> AsyncIterator[Callable[[datetime.timedelta, Callable], Awaitable[asyncio.Task]]]: +) -> AsyncIterator[ + Callable[ + [datetime.timedelta, Callable, asyncio.Event | None], Awaitable[asyncio.Task] + ] +]: created_tasks = [] async def _creator( - interval: datetime.timedelta, task: Callable[..., Awaitable] + interval: datetime.timedelta, + task: Callable[..., Awaitable], + early_wake_up_event: asyncio.Event | None, ) -> asyncio.Task: background_task = start_periodic_task( task, interval=interval, task_name=faker.pystr(), + early_wake_up_event=early_wake_up_event, ) assert background_task created_tasks.append(background_task) @@ -62,33 +75,69 @@ async def _creator( ) +@pytest.mark.parametrize( + "wake_up_event", [None, asyncio.Event], ids=lambda x: f"wake-up-event: {x}" +) async def test_background_task_created_and_deleted( mock_background_task: mock.AsyncMock, task_interval: datetime.timedelta, create_background_task: Callable[ - [datetime.timedelta, Callable], Awaitable[asyncio.Task] + [datetime.timedelta, Callable, asyncio.Event | None], Awaitable[asyncio.Task] ], + wake_up_event: Callable | None, ): - task = await create_background_task( + event = wake_up_event() if wake_up_event else None + _task = await create_background_task( task_interval, mock_background_task, + event, ) await asyncio.sleep(5 * task_interval.total_seconds()) mock_background_task.assert_called() - assert mock_background_task.call_count > 1 + assert mock_background_task.call_count > 2 + + +async def test_background_task_wakes_up_early( + mock_background_task: mock.AsyncMock, + very_long_task_interval: datetime.timedelta, + create_background_task: Callable[ + [datetime.timedelta, Callable, asyncio.Event | None], Awaitable[asyncio.Task] + ], +): + wake_up_event = asyncio.Event() + _task = await create_background_task( + very_long_task_interval, + mock_background_task, + wake_up_event, + ) + await asyncio.sleep(5 * _FAST_POLL_INTERVAL) + # now the task should have run only once + mock_background_task.assert_called_once() + await asyncio.sleep(5 * _FAST_POLL_INTERVAL) + mock_background_task.assert_called_once() + # this should wake up the task + wake_up_event.set() + await asyncio.sleep(5 * _FAST_POLL_INTERVAL) + mock_background_task.assert_called() + assert mock_background_task.call_count == 2 + # no change this now waits again a very long time + await asyncio.sleep(5 * _FAST_POLL_INTERVAL) + mock_background_task.assert_called() + assert mock_background_task.call_count == 2 async def test_background_task_raises_restarts( mock_background_task: mock.AsyncMock, task_interval: datetime.timedelta, create_background_task: Callable[ - [datetime.timedelta, Callable], Awaitable[asyncio.Task] + [datetime.timedelta, Callable, asyncio.Event | None], Awaitable[asyncio.Task] ], ): mock_background_task.side_effect = RuntimeError("pytest faked runtime error") - task = await create_background_task( + _task = await create_background_task( task_interval, mock_background_task, + None, ) await asyncio.sleep(5 * task_interval.total_seconds()) mock_background_task.assert_called() @@ -99,13 +148,14 @@ async def test_background_task_correctly_cancels( mock_background_task: mock.AsyncMock, task_interval: datetime.timedelta, create_background_task: Callable[ - [datetime.timedelta, Callable], Awaitable[asyncio.Task] + [datetime.timedelta, Callable, asyncio.Event | None], Awaitable[asyncio.Task] ], ): mock_background_task.side_effect = asyncio.CancelledError - task = await create_background_task( + _task = await create_background_task( task_interval, mock_background_task, + None, ) await asyncio.sleep(5 * task_interval.total_seconds()) # the task will be called once, and then stop diff --git a/services/director-v2/src/simcore_service_director_v2/api/dependencies/scheduler.py b/services/director-v2/src/simcore_service_director_v2/api/dependencies/scheduler.py index cd661759bc5..a0903608789 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/dependencies/scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/api/dependencies/scheduler.py @@ -1,7 +1,7 @@ from fastapi import Depends, FastAPI, Request from ...core.settings import ComputationalBackendSettings -from ...modules.comp_scheduler.base_scheduler import BaseCompScheduler +from ...modules.comp_scheduler import BaseCompScheduler from . import get_app diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py index 72bdf37e6c7..b7f47b186e7 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py @@ -63,7 +63,7 @@ from ...models.comp_runs import CompRunsAtDB, ProjectMetadataDict, RunMetadataDict from ...models.comp_tasks import CompTaskAtDB from ...modules.catalog import CatalogClient -from ...modules.comp_scheduler.base_scheduler import BaseCompScheduler +from ...modules.comp_scheduler import BaseCompScheduler from ...modules.db.repositories.clusters import ClustersRepository from ...modules.db.repositories.comp_pipelines import CompPipelinesRepository from ...modules.db.repositories.comp_runs import CompRunsRepository @@ -288,7 +288,7 @@ async def _try_start_pipeline( ) # NOTE: in case of a burst of calls to that endpoint, we might end up in a weird state. @run_sequentially_in_context(target_args=["computation.project_id"]) -async def create_computation( # noqa: PLR0913 # pylint:disable=too-many-positional-arguments +async def create_computation( # noqa: PLR0913 # pylint: disable=too-many-positional-arguments computation: ComputationCreate, request: Request, project_repo: Annotated[ diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 9972a42cce5..f1c81f18f98 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -173,9 +173,9 @@ def init_app(settings: AppSettings | None = None) -> FastAPI: ) if dynamic_scheduler_enabled or computational_backend_enabled: rabbitmq.setup(app) + redis.setup(app) if dynamic_scheduler_enabled: - redis.setup(app) dynamic_sidecar.setup(app) socketio.setup(app) notifier.setup(app) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/__init__.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/__init__.py index 1f9222007c5..1eb6c3dab10 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/__init__.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/__init__.py @@ -1,3 +1,15 @@ -from .background_task import setup +from fastapi import FastAPI -__all__: tuple[str, ...] = ("setup",) +from ._base_scheduler import BaseCompScheduler +from ._task import on_app_shutdown, on_app_startup + + +def setup(app: FastAPI): + app.add_event_handler("startup", on_app_startup(app)) + app.add_event_handler("shutdown", on_app_shutdown(app)) + + +__all__: tuple[str, ...] = ( + "setup", + "BaseCompScheduler", +) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/base_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_base_scheduler.py similarity index 100% rename from services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/base_scheduler.py rename to services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_base_scheduler.py diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_dask_scheduler.py similarity index 96% rename from services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py rename to services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_dask_scheduler.py index 3890ee1f7ad..51fb3b1a3fb 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_dask_scheduler.py @@ -30,10 +30,6 @@ from ...models.comp_runs import RunMetadataDict from ...models.comp_tasks import CompTaskAtDB from ...models.dask_subsystem import DaskClientTaskState -from ...modules.dask_client import DaskClient, PublishedComputationTask -from ...modules.dask_clients_pool import DaskClientsPool -from ...modules.db.repositories.clusters import ClustersRepository -from ...modules.db.repositories.comp_runs import CompRunsRepository from ...utils.comp_scheduler import Iteration, get_resource_tracking_run_id from ...utils.dask import ( clean_task_output_and_log_files_if_invalid, @@ -48,8 +44,12 @@ publish_service_stopped_metrics, ) from ..clusters_keeper import get_or_create_on_demand_cluster +from ..dask_client import DaskClient, PublishedComputationTask +from ..dask_clients_pool import DaskClientsPool +from ..db.repositories.clusters import ClustersRepository +from ..db.repositories.comp_runs import CompRunsRepository from ..db.repositories.comp_tasks import CompTasksRepository -from .base_scheduler import BaseCompScheduler, ScheduledPipelineParams +from ._base_scheduler import BaseCompScheduler, ScheduledPipelineParams _logger = logging.getLogger(__name__) @@ -158,9 +158,11 @@ async def _get_tasks_status( for dask_task_state, task in zip(tasks_statuses, tasks, strict=True): if dask_task_state is DaskClientTaskState.PENDING_OR_STARTED: running_states += [ - RunningState.STARTED - if task.progress is not None - else RunningState.PENDING + ( + RunningState.STARTED + if task.progress is not None + else RunningState.PENDING + ) ] else: running_states += [ diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/factory.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_factory.py similarity index 79% rename from services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/factory.py rename to services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_factory.py index 756fda1cb77..458950e9798 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/factory.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_factory.py @@ -2,16 +2,16 @@ from fastapi import FastAPI from models_library.clusters import DEFAULT_CLUSTER_ID -from simcore_service_director_v2.core.settings import AppSettings from ...core.errors import ConfigurationError +from ...core.settings import AppSettings from ...models.comp_runs import CompRunsAtDB -from ...modules.dask_clients_pool import DaskClientsPool -from ...modules.rabbitmq import get_rabbitmq_client, get_rabbitmq_rpc_client from ...utils.comp_scheduler import SCHEDULED_STATES +from ..dask_clients_pool import DaskClientsPool from ..db.repositories.comp_runs import CompRunsRepository -from .base_scheduler import BaseCompScheduler, ScheduledPipelineParams -from .dask_scheduler import DaskScheduler +from ..rabbitmq import get_rabbitmq_client, get_rabbitmq_rpc_client +from ._base_scheduler import BaseCompScheduler, ScheduledPipelineParams +from ._dask_scheduler import DaskScheduler logger = logging.getLogger(__name__) @@ -43,9 +43,9 @@ async def create_from_db(app: FastAPI) -> BaseCompScheduler: db_engine=db_engine, scheduled_pipelines={ (r.user_id, r.project_uuid, r.iteration): ScheduledPipelineParams( - cluster_id=r.cluster_id - if r.cluster_id is not None - else DEFAULT_CLUSTER_ID, + cluster_id=( + r.cluster_id if r.cluster_id is not None else DEFAULT_CLUSTER_ID + ), run_metadata=r.metadata, mark_for_cancellation=False, use_on_demand_clusters=r.use_on_demand_clusters, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_task.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_task.py new file mode 100644 index 00000000000..0e1c79ff8b6 --- /dev/null +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_task.py @@ -0,0 +1,50 @@ +import datetime +import logging +from collections.abc import Callable, Coroutine +from typing import Any, Final + +from fastapi import FastAPI +from servicelib.background_task import start_periodic_task, stop_periodic_task +from servicelib.logging_utils import log_context +from servicelib.redis import RedisClientsManager +from servicelib.redis_utils import exclusive +from settings_library.redis import RedisDatabase + +from . import _scheduler_factory + +_logger = logging.getLogger(__name__) + +_COMPUTATIONAL_SCHEDULER_INTERVAL: Final[datetime.timedelta] = datetime.timedelta( + seconds=5 +) +_TASK_NAME: Final[str] = "computational services scheduler" + + +def on_app_startup(app: FastAPI) -> Callable[[], Coroutine[Any, Any, None]]: + async def start_scheduler() -> None: + with log_context( + _logger, level=logging.INFO, msg="starting computational scheduler" + ): + redis_clients_manager: RedisClientsManager = app.state.redis_clients_manager + lock_key = f"{app.title}:computational_scheduler" + app.state.scheduler = scheduler = await _scheduler_factory.create_from_db( + app + ) + app.state.computational_scheduler_task = start_periodic_task( + exclusive( + redis_clients_manager.client(RedisDatabase.LOCKS), + lock_key=lock_key, + )(scheduler.schedule_all_pipelines), + interval=_COMPUTATIONAL_SCHEDULER_INTERVAL, + task_name=_TASK_NAME, + early_wake_up_event=scheduler.wake_up_event, + ) + + return start_scheduler + + +def on_app_shutdown(app: FastAPI) -> Callable[[], Coroutine[Any, Any, None]]: + async def stop_scheduler() -> None: + await stop_periodic_task(app.state.computational_scheduler_task) + + return stop_scheduler diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/background_task.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/background_task.py deleted file mode 100644 index b6fc18efbce..00000000000 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/background_task.py +++ /dev/null @@ -1,73 +0,0 @@ -import asyncio -import logging -from asyncio import CancelledError -from contextlib import suppress -from typing import Any, Callable, Coroutine - -from fastapi import FastAPI - -from . import factory - -logger = logging.getLogger(__name__) - -_DEFAULT_TIMEOUT_S: int = 5 - - -async def scheduler_task(app: FastAPI) -> None: - scheduler = app.state.scheduler - while app.state.comp_scheduler_running: - try: - logger.debug("Computational scheduler task running...") - await scheduler.schedule_all_pipelines() - with suppress(asyncio.TimeoutError): - await asyncio.wait_for( - scheduler.wake_up_event.wait(), timeout=_DEFAULT_TIMEOUT_S - ) - except CancelledError: - logger.info("Computational scheduler task cancelled") - raise - except Exception: # pylint: disable=broad-except - if not app.state.comp_scheduler_running: - logger.warning("Forced to stop computational scheduler") - break - logger.exception( - "Unexpected error in computational scheduler task, restarting scheduler now..." - ) - # wait a bit before restarting the task - await asyncio.sleep(_DEFAULT_TIMEOUT_S) - - -def on_app_startup(app: FastAPI) -> Callable[[], Coroutine[Any, Any, None]]: - async def start_scheduler() -> None: - # FIXME: added this variable to overcome the state in which the - # task cancelation is ignored and the exceptions enter in a loop - # that never stops the background task. This flag is an additional - # mechanism to enforce stopping the background task - app.state.comp_scheduler_running = True - app.state.scheduler = await factory.create_from_db(app) - app.state.scheduler_task = asyncio.create_task( - scheduler_task(app), name="computational services scheduler" - ) - logger.info("Computational services Scheduler started") - - return start_scheduler - - -def on_app_shutdown(app: FastAPI) -> Callable[[], Coroutine[Any, Any, None]]: - async def stop_scheduler() -> None: - logger.info("Computational services Scheduler stopping...") - task = app.state.scheduler_task - with suppress(CancelledError): - app.state.comp_scheduler_running = False - task.cancel() - await task - app.state.scheduler = None - app.state.scheduler_task = None - logger.info("Computational services Scheduler stopped") - - return stop_scheduler - - -def setup(app: FastAPI): - app.add_event_handler("startup", on_app_startup(app)) - app.add_event_handler("shutdown", on_app_shutdown(app)) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 110dbd5f89b..16a6311da1b 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -32,6 +32,7 @@ from pytest_simcore.helpers.postgres_tools import PostgresTestConfig from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.rabbit import RabbitSettings +from settings_library.redis import RedisSettings from starlette import status from starlette.testclient import TestClient from yarl import URL @@ -87,6 +88,7 @@ def minimal_configuration( postgres_db: sa.engine.Engine, postgres_host_config: PostgresTestConfig, rabbit_service: RabbitSettings, + redis_service: RedisSettings, simcore_services_ready: None, storage_service: URL, ) -> None: diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 17d3fe4bcca..720e7d0c3e1 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -108,6 +108,7 @@ "migration", "postgres", "rabbit", + "redis", "storage", "redis", ] diff --git a/services/director-v2/tests/unit/_helpers.py b/services/director-v2/tests/unit/_helpers.py index 60eea950f12..2654c63a3e1 100644 --- a/services/director-v2/tests/unit/_helpers.py +++ b/services/director-v2/tests/unit/_helpers.py @@ -11,7 +11,7 @@ from simcore_service_director_v2.models.comp_pipelines import CompPipelineAtDB from simcore_service_director_v2.models.comp_runs import CompRunsAtDB from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB -from simcore_service_director_v2.modules.comp_scheduler.base_scheduler import ( +from simcore_service_director_v2.modules.comp_scheduler._base_scheduler import ( BaseCompScheduler, ) @@ -30,7 +30,7 @@ class RunningProject(PublishedProject): async def trigger_comp_scheduler(scheduler: BaseCompScheduler) -> None: # trigger the scheduler - scheduler._wake_up_scheduler_now() # pylint: disable=protected-access + scheduler._wake_up_scheduler_now() # pylint: disable=protected-access # noqa: SLF001 # let the scheduler be actually triggered await asyncio.sleep(1) diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py index 81034fbaee5..1135465ef61 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py @@ -53,6 +53,7 @@ from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.rabbit import RabbitSettings +from settings_library.redis import RedisSettings from simcore_postgres_database.models.comp_pipeline import StateType from simcore_postgres_database.models.comp_tasks import NodeClass from simcore_postgres_database.utils_projects_nodes import ProjectNodesRepo @@ -65,7 +66,7 @@ ) from simcore_service_director_v2.utils.computations import to_node_class -pytest_simcore_core_services_selection = ["postgres", "rabbit"] +pytest_simcore_core_services_selection = ["postgres", "rabbit", "redis"] pytest_simcore_ops_services_selection = [ "adminer", ] @@ -84,6 +85,7 @@ def minimal_configuration( mock_env: EnvVarsDict, postgres_host_config: dict[str, str], rabbit_service: RabbitSettings, + redis_service: RedisSettings, monkeypatch: pytest.MonkeyPatch, mocked_rabbit_mq_client: None, faker: Faker, diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py index d15ab46a498..fbc90204f83 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py @@ -46,6 +46,7 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQClient from settings_library.rabbit import RabbitSettings +from settings_library.redis import RedisSettings from simcore_postgres_database.models.comp_runs import comp_runs from simcore_postgres_database.models.comp_tasks import NodeClass, comp_tasks from simcore_service_director_v2.core.application import init_app @@ -65,11 +66,10 @@ from simcore_service_director_v2.models.comp_runs import CompRunsAtDB, RunMetadataDict from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB, Image from simcore_service_director_v2.models.dask_subsystem import DaskClientTaskState -from simcore_service_director_v2.modules.comp_scheduler import background_task -from simcore_service_director_v2.modules.comp_scheduler.base_scheduler import ( +from simcore_service_director_v2.modules.comp_scheduler._base_scheduler import ( BaseCompScheduler, ) -from simcore_service_director_v2.modules.comp_scheduler.dask_scheduler import ( +from simcore_service_director_v2.modules.comp_scheduler._dask_scheduler import ( DaskScheduler, ) from simcore_service_director_v2.modules.dask_client import ( @@ -84,7 +84,7 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -pytest_simcore_core_services_selection = ["postgres", "rabbit"] +pytest_simcore_core_services_selection = ["postgres", "rabbit", "redis"] pytest_simcore_ops_services_selection = [ "adminer", ] @@ -103,7 +103,9 @@ def _assert_dask_client_correctly_initialized( ) mocked_dask_client.register_handlers.assert_called_once_with( TaskHandlers( - cast(DaskScheduler, scheduler)._task_progress_change_handler, + cast( + DaskScheduler, scheduler + )._task_progress_change_handler, # noqa: SLF001 cast(DaskScheduler, scheduler)._task_log_change_handler, # noqa: SLF001 ) ) @@ -163,6 +165,7 @@ def minimal_dask_scheduler_config( postgres_host_config: dict[str, str], monkeypatch: pytest.MonkeyPatch, rabbit_service: RabbitSettings, + redis_service: RedisSettings, faker: Faker, ) -> None: """set a minimal configuration for testing the dask connection only""" @@ -202,7 +205,7 @@ def mocked_dask_client(mocker: MockerFixture) -> mock.MagicMock: @pytest.fixture def mocked_parse_output_data_fct(mocker: MockerFixture) -> mock.Mock: return mocker.patch( - "simcore_service_director_v2.modules.comp_scheduler.dask_scheduler.parse_output_data", + "simcore_service_director_v2.modules.comp_scheduler._dask_scheduler.parse_output_data", autospec=True, ) @@ -210,7 +213,7 @@ def mocked_parse_output_data_fct(mocker: MockerFixture) -> mock.Mock: @pytest.fixture def mocked_clean_task_output_fct(mocker: MockerFixture) -> mock.MagicMock: return mocker.patch( - "simcore_service_director_v2.modules.comp_scheduler.dask_scheduler.clean_task_output_and_log_files_if_invalid", + "simcore_service_director_v2.modules.comp_scheduler._dask_scheduler.clean_task_output_and_log_files_if_invalid", return_value=None, autospec=True, ) @@ -219,7 +222,15 @@ def mocked_clean_task_output_fct(mocker: MockerFixture) -> mock.MagicMock: @pytest.fixture def with_disabled_scheduler_task(mocker: MockerFixture) -> None: """disables the scheduler task, note that it needs to be triggered manually then""" - mocker.patch.object(background_task, "scheduler_task") + mocker.patch( + "simcore_service_director_v2.modules.comp_scheduler._task.start_periodic_task", + autospec=True, + ) + + mocker.patch( + "simcore_service_director_v2.modules.comp_scheduler._task.stop_periodic_task", + autospec=True, + ) @pytest.fixture @@ -229,13 +240,13 @@ async def minimal_app(async_client: httpx.AsyncClient) -> FastAPI: # a new thread on which it creates a new loop # causing issues downstream with coroutines not # being created on the same loop - return async_client._transport.app # type: ignore + return async_client._transport.app # type: ignore # noqa: SLF001 @pytest.fixture def mocked_clean_task_output_and_log_files_if_invalid(mocker: MockerFixture) -> None: mocker.patch( - "simcore_service_director_v2.modules.comp_scheduler.dask_scheduler.clean_task_output_and_log_files_if_invalid", + "simcore_service_director_v2.modules.comp_scheduler._dask_scheduler.clean_task_output_and_log_files_if_invalid", autospec=True, ) @@ -247,7 +258,7 @@ async def test_scheduler_gracefully_starts_and_stops( minimal_app: FastAPI, ): # check it started correctly - assert minimal_app.state.scheduler_task is not None + assert minimal_app.state.computational_scheduler_task is not None @pytest.mark.parametrize( @@ -1055,7 +1066,7 @@ async def test_handling_of_disconnected_dask_scheduler( ): # this will create a non connected backend issue that will trigger re-connection mocked_dask_client_send_task = mocker.patch( - "simcore_service_director_v2.modules.comp_scheduler.dask_scheduler.DaskClient.send_computation_tasks", + "simcore_service_director_v2.modules.comp_scheduler._dask_scheduler.DaskClient.send_computation_tasks", side_effect=backend_error, ) assert mocked_dask_client_send_task @@ -1360,7 +1371,7 @@ async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskSta @pytest.fixture async def mocked_get_or_create_cluster(mocker: MockerFixture) -> mock.Mock: return mocker.patch( - "simcore_service_director_v2.modules.comp_scheduler.dask_scheduler.get_or_create_on_demand_cluster", + "simcore_service_director_v2.modules.comp_scheduler._dask_scheduler.get_or_create_on_demand_cluster", autospec=True, ) diff --git a/services/director/requirements/_tools.in b/services/director/requirements/_tools.in index 05f1ab1646f..e69de29bb2d 100644 --- a/services/director/requirements/_tools.in +++ b/services/director/requirements/_tools.in @@ -1,7 +0,0 @@ ---constraint _base.txt ---constraint _test.txt - -watchdog[watchmedo] -black~=20.8b0 -pip-tools -bump2version diff --git a/services/director/requirements/_tools.txt b/services/director/requirements/_tools.txt index 821e63f1a10..38ed7220aed 100644 --- a/services/director/requirements/_tools.txt +++ b/services/director/requirements/_tools.txt @@ -1,3 +1,9 @@ +# +# This file is autogenerated by pip-compile with python 3.6 +# To update, run: +# +# pip-compile --output-file=requirements/_tools.txt --strip-extras requirements/_tools.in +# appdirs==1.4.4 # via black black==20.8b1 @@ -8,14 +14,22 @@ click==8.0.3 # via # black # pip-tools +dataclasses==0.7 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # black +importlib-metadata==2.0.0 + # via + # -c requirements/_test.txt + # click + # pep517 mypy-extensions==0.4.3 # via black pathspec==0.9.0 # via black pep517==0.12.0 # via pip-tools -pip==24.2 - # via pip-tools pip-tools==6.4.0 # via -r requirements/_tools.in pyyaml==5.4 @@ -25,8 +39,6 @@ pyyaml==5.4 # watchdog regex==2022.1.18 # via black -setuptools==75.2.0 - # via pip-tools toml==0.10.2 # via # -c requirements/_test.txt @@ -43,3 +55,12 @@ watchdog==2.1.6 # via -r requirements/_tools.in wheel==0.37.1 # via pip-tools +zipp==3.4.0 + # via + # -c requirements/_test.txt + # importlib-metadata + # pep517 + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools From cddaab909b4ae27cefa255f7c6970dd43c14d679 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:31:07 +0100 Subject: [PATCH 06/22] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Mark=20new=20trash?= =?UTF-8?q?=20web-api=20as=20dev-features=20(#6665)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .codecov.yml | 1 + .../projects/_trash_handlers.py | 4 ++++ .../web/server/tests/unit/with_dbs/03/test_trash.py | 11 +++++++++++ 3 files changed, 16 insertions(+) diff --git a/.codecov.yml b/.codecov.yml index 6e9caac4b87..341e18a09bd 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -18,6 +18,7 @@ flag_management: component_management: default_rules: + carryforward: true statuses: - type: project target: auto diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py index 5defa97c927..2995488c562 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py @@ -14,6 +14,7 @@ from servicelib.status_codes_utils import is_5xx_server_error from .._meta import API_VTAG as VTAG +from ..application_settings_utils import requires_dev_feature_enabled from ..login.decorators import get_user_id, login_required from ..products.api import get_product_name from ..projects._common_models import ProjectPathParams @@ -102,6 +103,7 @@ async def _wrapper(request: web.Request) -> web.StreamResponse: @routes.delete(f"/{VTAG}/trash", name="empty_trash") +@requires_dev_feature_enabled @login_required @permission_required("project.delete") @_handle_request_exceptions @@ -117,6 +119,7 @@ async def empty_trash(request: web.Request): @routes.post(f"/{VTAG}/projects/{{project_id}}:trash", name="trash_project") +@requires_dev_feature_enabled @login_required @permission_required("project.delete") @_handle_request_exceptions @@ -140,6 +143,7 @@ async def trash_project(request: web.Request): @routes.post(f"/{VTAG}/projects/{{project_id}}:untrash", name="untrash_project") +@requires_dev_feature_enabled @login_required @permission_required("project.delete") @_handle_request_exceptions diff --git a/services/web/server/tests/unit/with_dbs/03/test_trash.py b/services/web/server/tests/unit/with_dbs/03/test_trash.py index e10dc0065f1..5a760d5f9fd 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_trash.py +++ b/services/web/server/tests/unit/with_dbs/03/test_trash.py @@ -18,12 +18,23 @@ from models_library.rest_pagination import Page from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects.models import ProjectDict +@pytest.fixture +def app_environment( + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch +) -> EnvVarsDict: + return app_environment | setenvs_from_dict( + monkeypatch, {"WEBSERVER_DEV_FEATURES_ENABLED": "1"} + ) + + @pytest.fixture def user_role() -> UserRole: return UserRole.USER From 41817c80d8808f9aa57a69758247f936ecaa5b1e Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:55:11 +0100 Subject: [PATCH 07/22] =?UTF-8?q?=F0=9F=8E=A8=20[e2e-frontend]=20Service?= =?UTF-8?q?=20browser=20(#6664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../class/osparc/dashboard/ResourceFilter.js | 8 +- .../tests/serviceBrowser/leftFilters.spec.js | 64 ++++++++++++++ .../tests/serviceBrowser/mainView.spec.js | 88 +++++++++++++++++++ .../tests/studyBrowser/leftFilters.spec.js | 4 +- 4 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 tests/e2e-frontend/tests/serviceBrowser/leftFilters.spec.js create mode 100644 tests/e2e-frontend/tests/serviceBrowser/mainView.spec.js diff --git a/services/static-webserver/client/source/class/osparc/dashboard/ResourceFilter.js b/services/static-webserver/client/source/class/osparc/dashboard/ResourceFilter.js index 0cbb3357737..142cdab7d3f 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/ResourceFilter.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/ResourceFilter.js @@ -22,7 +22,7 @@ qx.Class.define("osparc.dashboard.ResourceFilter", { construct: function(resourceType) { this.base(arguments); - osparc.utils.Utils.setIdToWidget(this, "resourceFilter"); + osparc.utils.Utils.setIdToWidget(this, resourceType + "-resourceFilter"); this.__resourceType = resourceType; this.__sharedWithButtons = []; @@ -102,6 +102,7 @@ qx.Class.define("osparc.dashboard.ResourceFilter", { label: option.label, icon: option.icon, }); + osparc.utils.Utils.setIdToWidget(button, this.__resourceType + "-sharedWithFilterItem"); if (this.__resourceType === "study") { if (option.id === "show-all") { button.set({ @@ -138,7 +139,7 @@ qx.Class.define("osparc.dashboard.ResourceFilter", { /* TAGS */ __createTagsFilterLayout: function() { const layout = new qx.ui.container.Composite(new qx.ui.layout.VBox(5)); - osparc.utils.Utils.setIdToWidget(layout, "tagsFilter"); + osparc.utils.Utils.setIdToWidget(layout, this.__resourceType + "-tagsFilter"); this.__populateTags(layout, []); osparc.store.Store.getInstance().addListener("changeTags", () => { @@ -159,7 +160,7 @@ qx.Class.define("osparc.dashboard.ResourceFilter", { layout.removeAll(); osparc.store.Store.getInstance().getTags().forEach((tag, idx) => { const button = new qx.ui.form.ToggleButton(tag.name, "@FontAwesome5Solid/tag/18"); - osparc.utils.Utils.setIdToWidget(button, "tagFilterItem"); + osparc.utils.Utils.setIdToWidget(button, this.__resourceType + "-tagFilterItem"); button.id = tag.id; button.set({ appearance: "filter-toggle-button", @@ -220,6 +221,7 @@ qx.Class.define("osparc.dashboard.ResourceFilter", { const iconSize = 20; const button = new qx.ui.toolbar.RadioButton(serviceType.label, serviceType.icon+iconSize); button.id = serviceId; + osparc.utils.Utils.setIdToWidget(button, this.__resourceType + "-serviceTypeFilterItem"); button.set({ appearance: "filter-toggle-button", value: false diff --git a/tests/e2e-frontend/tests/serviceBrowser/leftFilters.spec.js b/tests/e2e-frontend/tests/serviceBrowser/leftFilters.spec.js new file mode 100644 index 00000000000..247f5def582 --- /dev/null +++ b/tests/e2e-frontend/tests/serviceBrowser/leftFilters.spec.js @@ -0,0 +1,64 @@ +/* eslint-disable no-undef */ + +const { test, expect } = require('@playwright/test'); + +import { LoginPage } from '../fixtures/loginPage'; + +import products from '../products.json'; +import users from '../users.json'; + +const product = "osparc"; +const productUrl = products[product]; +const user = users[product][0]; + +test.describe.serial(`Left Filters:`, () => { + let page = null; + let loginPageFixture = null; + + test.beforeAll(async ({ browser }) => { + page = await browser.newPage(); + + const responsePromise = page.waitForResponse('**/services/-/latest**', { + timeout: 30000 + }); + + loginPageFixture = new LoginPage(page, productUrl); + const role = await loginPageFixture.login(user.email, user.password); + expect(role).toBe(user.role); + + await responsePromise; + + await page.getByTestId("servicesTabBtn").click(); + }); + + test.afterAll(async ({ browser }) => { + await loginPageFixture.logout(); + await page.close(); + await browser.close(); + }); + + test(`Filters`, async () => { + const sharedWithButtons = page.getByTestId("service-sharedWithFilterItem"); + await expect(sharedWithButtons.first()).toBeVisible({ + timeout: 30000 // it will take some time to load the Study Browser + }); + + const countSharedWithButtons = await sharedWithButtons.count(); + // All Services + // My Services + // Shared with Me + // Shared with Everyone + expect(countSharedWithButtons === 4).toBeTruthy(); + + + const serviceTypeButtons = page.getByTestId("service-serviceTypeFilterItem"); + await expect(serviceTypeButtons.first()).toBeVisible({ + timeout: 30000 // it will take some time to load the Study Browser + }); + + const countServiceTypeButtons = await serviceTypeButtons.count(); + // Computational + // Interactive + expect(countServiceTypeButtons === 2).toBeTruthy(); + }); +}); diff --git a/tests/e2e-frontend/tests/serviceBrowser/mainView.spec.js b/tests/e2e-frontend/tests/serviceBrowser/mainView.spec.js new file mode 100644 index 00000000000..c3077e467e7 --- /dev/null +++ b/tests/e2e-frontend/tests/serviceBrowser/mainView.spec.js @@ -0,0 +1,88 @@ +/* eslint-disable no-undef */ + +const { test, expect } = require('@playwright/test'); + +import { LoginPage } from '../fixtures/loginPage'; + +import products from '../products.json'; +import users from '../users.json'; + +const servicesTabExposed = { + "osparc": { + "areServicesExposed": true, + }, + "s4l": { + "areServicesExposed": true, + }, + "s4lacad": { + "areServicesExposed": true, + }, + "s4llite": { + "areServicesExposed": false, + }, + "tis": { + "areServicesExposed": false, + }, + "tiplite": { + "areServicesExposed": false, + }, +} + +for (const product in products) { + expect(servicesTabExposed[product]).toBeDefined(); + if (!servicesTabExposed[product]["areServicesExposed"]) { + continue; + } + + if (product in users) { + const productUrl = products[product]; + const productUsers = users[product]; + for (const user of productUsers) { + // expected roles for users: "USER" + const role = "USER"; + expect(user.role).toBe(role); + + test.describe.serial(`Main View: ${product}`, () => { + let page = null; + let loginPageFixture = null; + + test.beforeAll(async ({ browser }) => { + page = await browser.newPage(); + + const responsePromise = page.waitForResponse('**/services/-/latest**', { + timeout: 30000 + }); + + loginPageFixture = new LoginPage(page, productUrl); + const role = await loginPageFixture.login(user.email, user.password); + expect(role).toBe(user.role); + + const response = await responsePromise; + const resp = await response.json(); + expect("data" in resp && "_meta" in resp["data"] && "total" in resp["data"]["_meta"]); + console.log("N Services in Response:", resp["data"]["_meta"]["total"]); + + await page.getByTestId("servicesTabBtn").click(); + }); + + test.afterAll(async ({ browser }) => { + await loginPageFixture.logout(); + await page.close(); + await browser.close(); + }); + + test(`Services list`, async () => { + const servicesList = page.getByTestId("servicesList"); + await expect(servicesList).toBeVisible({ + timeout: 30000 + }); + + const serviceCards = servicesList.locator(':scope > *'); + const count = await serviceCards.count(); + console.log("N Services listed", count); + expect(count > 0); + }); + }); + } + } +} diff --git a/tests/e2e-frontend/tests/studyBrowser/leftFilters.spec.js b/tests/e2e-frontend/tests/studyBrowser/leftFilters.spec.js index 2a8dfa724d7..5412b82baae 100644 --- a/tests/e2e-frontend/tests/studyBrowser/leftFilters.spec.js +++ b/tests/e2e-frontend/tests/studyBrowser/leftFilters.spec.js @@ -41,12 +41,12 @@ test.describe.serial(`Left Filters:`, () => { }); test(`Tags`, async () => { - const tagsFilter = page.getByTestId("tagsFilter"); + const tagsFilter = page.getByTestId("study-tagsFilter"); await expect(tagsFilter).toBeVisible({ timeout: 30000 // it will take some time to load the Study Browser }); - const tagFilterItems = page.getByTestId("tagFilterItem"); + const tagFilterItems = page.getByTestId("study-tagFilterItem"); const count = await tagFilterItems.count(); // at least two and less than 6 (max five are shown) expect(count > 1 && count < 6).toBeTruthy(); From d8f172080da94d6f57e77af3e5a30f8fb4a8be5a Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Tue, 5 Nov 2024 13:08:52 +0100 Subject: [PATCH 08/22] =?UTF-8?q?=F0=9F=90=9B=20Avoids=20raising=20error?= =?UTF-8?q?=20when=20composing=20disk=20usage=20(#6660)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../modules/system_monitor/_disk_usage.py | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/system_monitor/_disk_usage.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/system_monitor/_disk_usage.py index cdaa531aeb8..f1f34e0a6e7 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/system_monitor/_disk_usage.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/system_monitor/_disk_usage.py @@ -52,10 +52,6 @@ def _get_normalized_folder_name(path: Path) -> str: return f"{path}".replace("/", "_") -def _have_common_entries(a: set[str], b: set[str]) -> bool: - return bool(len(a & b) > 0) - - @dataclass class DiskUsageMonitor: app: FastAPI @@ -124,11 +120,11 @@ def _replace_incoming_usage( @staticmethod def _get_grouped_usage_to_folder_names( - normalized_disk_usage: dict[str, DiskUsage] + local_disk_usage: dict[str, DiskUsage] ) -> dict[DiskUsage, set[str]]: """Groups all paths that have the same metrics together""" usage_to_folder_names: dict[DiskUsage, set[str]] = {} - for folder_name, disk_usage in normalized_disk_usage.items(): + for folder_name, disk_usage in local_disk_usage.items(): if disk_usage not in usage_to_folder_names: usage_to_folder_names[disk_usage] = set() @@ -170,18 +166,12 @@ async def _monitor(self) -> None: msg = f"Could not assign {disk_usage=} for {folder_names=}" raise RuntimeError(msg) - detected_items = set(usage.keys()) - if not detected_items.issubset(_SUPPORTED_ITEMS): - msg = ( - f"Computed {usage=}, has unsupported items {detected_items=}. " - f"Currently only the following are supported: {_SUPPORTED_ITEMS}" - ) - raise RuntimeError(msg) + supported_usage = {k: v for k, v in usage.items() if k in _SUPPORTED_ITEMS} # notify only when usage changes - if self._last_usage != usage: - await self._publish_disk_usage(usage) - self._last_usage = usage + if self._last_usage != supported_usage: + await self._publish_disk_usage(supported_usage) + self._last_usage = supported_usage async def setup(self) -> None: self._monitor_task = start_periodic_task( From cd20a2c3969338deddcf7aa58a50fada97ffedb4 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser <8209087+mrnicegyu11@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:21:27 +0100 Subject: [PATCH 09/22] =?UTF-8?q?=F0=9F=8E=A8=20Tracing:=20Add=20more=20au?= =?UTF-8?q?toinstrumentation,=20enhance=20`setup=5Ftracing()`=20-=20DON'T?= =?UTF-8?q?=20DELETE=20PR=20BRANCH=20(#6561)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: dependabot[bot] Co-authored-by: Dustin Kaiser Co-authored-by: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Co-authored-by: Andrei Neagu Co-authored-by: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Co-authored-by: Sylvain <35365065+sanderegg@users.noreply.github.com> Co-authored-by: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: matusdrobuliak66 <60785969+matusdrobuliak66@users.noreply.github.com> Co-authored-by: Julian Querido Co-authored-by: Odei Maiz --- packages/aws-library/requirements/_base.in | 1 + packages/aws-library/requirements/_base.txt | 19 ++++- .../postgres-database/requirements/_base.in | 1 + .../postgres-database/requirements/_base.txt | 25 ++++++ .../postgres-database/requirements/_tools.txt | 6 +- .../service-library/requirements/_base.in | 1 + .../service-library/requirements/_base.txt | 11 ++- .../service-library/requirements/_test.in | 2 + .../service-library/requirements/_test.txt | 8 ++ .../service-library/requirements/_tools.txt | 6 +- .../src/servicelib/aiohttp/tracing.py | 50 +++++++++-- .../src/servicelib/fastapi/tracing.py | 84 ++++++++++++++++++- .../tests/aiohttp/test_tracing.py | 67 +++++++++++++++ packages/service-library/tests/conftest.py | 57 +++++++++++++ .../tests/fastapi/test_tracing.py | 66 ++++++++++++++- .../tests/rabbitmq/test_rabbitmq.py | 6 +- packages/simcore-sdk/requirements/_base.in | 1 + packages/simcore-sdk/requirements/_base.txt | 26 +++++- services/agent/requirements/_base.txt | 8 ++ services/api-server/requirements/_base.txt | 53 +++++++++--- services/autoscaling/requirements/_base.txt | 18 ++++ services/autoscaling/requirements/_test.txt | 4 + services/autoscaling/tests/unit/conftest.py | 8 ++ ...test_modules_auto_scaling_computational.py | 1 + services/catalog/requirements/_base.txt | 41 +++++---- .../clusters-keeper/requirements/_base.txt | 18 ++++ .../clusters-keeper/requirements/_test.txt | 4 + services/dask-sidecar/requirements/_base.txt | 11 ++- .../datcore-adapter/requirements/_base.txt | 8 ++ services/director-v2/requirements/_base.txt | 53 +++++++++--- services/director/requirements/_tools.txt | 2 + .../dynamic-scheduler/requirements/_base.txt | 41 +++++---- .../dynamic-sidecar/requirements/_base.txt | 53 +++++++++--- services/efs-guardian/requirements/_base.txt | 25 +++++- services/efs-guardian/requirements/_test.txt | 4 + services/invitations/requirements/_base.txt | 8 ++ services/opentelemetry-collector-config.yaml | 8 +- services/payments/requirements/_base.txt | 41 +++++---- .../requirements/_base.txt | 25 +++++- .../requirements/_test.txt | 4 + services/storage/requirements/_base.in | 1 + services/storage/requirements/_base.txt | 24 ++++++ services/storage/requirements/_test.txt | 4 + .../simcore_service_storage/application.py | 1 - services/web/server/requirements/_base.txt | 47 +++++++---- services/web/server/requirements/_test.txt | 1 + .../src/simcore_service_webserver/tracing.py | 1 - 47 files changed, 826 insertions(+), 128 deletions(-) diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in index 628cebcf110..2cde3a8eeff 100644 --- a/packages/aws-library/requirements/_base.in +++ b/packages/aws-library/requirements/_base.in @@ -11,4 +11,5 @@ aiocache arrow pydantic[email] types-aiobotocore[ec2,s3,ssm] +opentelemetry-instrumentation-botocore sh diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index 9f07cc7025c..63c88ba0037 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -44,6 +44,8 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in +async-timeout==4.0.3 + # via redis attrs==24.2.0 # via # aiohttp @@ -127,7 +129,10 @@ opentelemetry-api==1.27.0 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation + # opentelemetry-instrumentation-botocore + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests + # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions opentelemetry-exporter-otlp==1.27.0 @@ -141,9 +146,18 @@ opentelemetry-exporter-otlp-proto-grpc==1.27.0 opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp opentelemetry-instrumentation==0.48b0 - # via opentelemetry-instrumentation-requests + # via + # opentelemetry-instrumentation-botocore + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests +opentelemetry-instrumentation-botocore==0.48b0 + # via -r requirements/_base.in +opentelemetry-instrumentation-redis==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-propagator-aws-xray==1.0.2 + # via opentelemetry-instrumentation-botocore opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common @@ -156,6 +170,8 @@ opentelemetry-sdk==1.27.0 # opentelemetry-exporter-otlp-proto-http opentelemetry-semantic-conventions==0.48b0 # via + # opentelemetry-instrumentation-botocore + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.48b0 @@ -297,6 +313,7 @@ wrapt==1.16.0 # aiobotocore # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.12.1 # via # aio-pika diff --git a/packages/postgres-database/requirements/_base.in b/packages/postgres-database/requirements/_base.in index 48679f44663..7a1a874c194 100644 --- a/packages/postgres-database/requirements/_base.in +++ b/packages/postgres-database/requirements/_base.in @@ -7,4 +7,5 @@ alembic pydantic sqlalchemy[postgresql_psycopg2binary,postgresql_asyncpg] # SEE extras in https://github.com/sqlalchemy/sqlalchemy/blob/main/setup.cfg#L43 +opentelemetry-instrumentation-asyncpg yarl diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt index bded83bb4f9..044a006295a 100644 --- a/packages/postgres-database/requirements/_base.txt +++ b/packages/postgres-database/requirements/_base.txt @@ -4,10 +4,16 @@ async-timeout==4.0.3 # via asyncpg asyncpg==0.29.0 # via sqlalchemy +deprecated==1.2.14 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions greenlet==3.1.1 # via sqlalchemy idna==3.10 # via yarl +importlib-metadata==8.4.0 + # via opentelemetry-api mako==1.3.5 # via # -c requirements/../../../requirements/constraints.txt @@ -16,12 +22,25 @@ markupsafe==2.1.5 # via mako multidict==6.1.0 # via yarl +opentelemetry-api==1.27.0 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asyncpg + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.48b0 + # via opentelemetry-instrumentation-asyncpg +opentelemetry-instrumentation-asyncpg==0.48b0 + # via -r requirements/_base.in +opentelemetry-semantic-conventions==0.48b0 + # via opentelemetry-instrumentation-asyncpg psycopg2-binary==2.9.9 # via sqlalchemy pydantic==1.10.18 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in +setuptools==75.2.0 + # via opentelemetry-instrumentation sqlalchemy==1.4.54 # via # -c requirements/../../../requirements/constraints.txt @@ -31,5 +50,11 @@ typing-extensions==4.12.2 # via # alembic # pydantic +wrapt==1.16.0 + # via + # deprecated + # opentelemetry-instrumentation yarl==1.12.1 # via -r requirements/_base.in +zipp==3.20.2 + # via importlib-metadata diff --git a/packages/postgres-database/requirements/_tools.txt b/packages/postgres-database/requirements/_tools.txt index d48ce38c32f..61c9a3ec7e1 100644 --- a/packages/postgres-database/requirements/_tools.txt +++ b/packages/postgres-database/requirements/_tools.txt @@ -68,8 +68,10 @@ pyyaml==6.0.2 # pre-commit ruff==0.6.7 # via -r requirements/../../../requirements/devenv.txt -setuptools==75.1.0 - # via pip-tools +setuptools==75.2.0 + # via + # -c requirements/_base.txt + # pip-tools tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 diff --git a/packages/service-library/requirements/_base.in b/packages/service-library/requirements/_base.in index aa776fedb15..295a76b0db8 100644 --- a/packages/service-library/requirements/_base.in +++ b/packages/service-library/requirements/_base.in @@ -18,6 +18,7 @@ faststream opentelemetry-api opentelemetry-exporter-otlp opentelemetry-instrumentation-requests +opentelemetry-instrumentation-redis opentelemetry-sdk psutil pydantic diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt index 2d0efb438c4..d53ce73a8c4 100644 --- a/packages/service-library/requirements/_base.txt +++ b/packages/service-library/requirements/_base.txt @@ -28,6 +28,8 @@ arrow==1.3.0 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/_base.in +async-timeout==4.0.3 + # via redis attrs==24.2.0 # via # aiohttp @@ -93,6 +95,7 @@ opentelemetry-api==1.27.0 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions @@ -107,7 +110,11 @@ opentelemetry-exporter-otlp-proto-grpc==1.27.0 opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp opentelemetry-instrumentation==0.48b0 - # via opentelemetry-instrumentation-requests + # via + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests +opentelemetry-instrumentation-redis==0.48b0 + # via -r requirements/_base.in opentelemetry-instrumentation-requests==0.48b0 # via -r requirements/_base.in opentelemetry-proto==1.27.0 @@ -122,6 +129,7 @@ opentelemetry-sdk==1.27.0 # opentelemetry-exporter-otlp-proto-http opentelemetry-semantic-conventions==0.48b0 # via + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.48b0 @@ -219,6 +227,7 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.12.1 # via # aio-pika diff --git a/packages/service-library/requirements/_test.in b/packages/service-library/requirements/_test.in index d3936487c75..239a389cbc0 100644 --- a/packages/service-library/requirements/_test.in +++ b/packages/service-library/requirements/_test.in @@ -13,6 +13,7 @@ # testing asgi_lifespan +botocore coverage docker faker @@ -20,6 +21,7 @@ flaky numpy openapi-spec-validator pillow +pip pytest pytest-aiohttp pytest-asyncio diff --git a/packages/service-library/requirements/_test.txt b/packages/service-library/requirements/_test.txt index f1679efbfdd..d622d382b4c 100644 --- a/packages/service-library/requirements/_test.txt +++ b/packages/service-library/requirements/_test.txt @@ -29,6 +29,8 @@ attrs==24.2.0 # jsonschema # pytest-docker # referencing +botocore==1.35.50 + # via -r requirements/_test.in certifi==2024.8.30 # via # -c requirements/../../../requirements/constraints.txt @@ -91,6 +93,8 @@ idna==3.10 # yarl iniconfig==2.0.0 # via pytest +jmespath==1.0.1 + # via botocore jsonschema==4.23.0 # via # -c requirements/_aiohttp.txt @@ -141,6 +145,8 @@ pathable==0.4.3 # jsonschema-path pillow==10.4.0 # via -r requirements/_test.in +pip==24.3.1 + # via -r requirements/_test.in pluggy==1.5.0 # via pytest pprintpp==0.4.0 @@ -188,6 +194,7 @@ pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 # via # -c requirements/_base.txt + # botocore # faker python-dotenv==1.0.1 # via -r requirements/_test.in @@ -263,6 +270,7 @@ urllib3==2.2.3 # -c requirements/../../../requirements/constraints.txt # -c requirements/_aiohttp.txt # -c requirements/_base.txt + # botocore # docker # requests yarl==1.12.1 diff --git a/packages/service-library/requirements/_tools.txt b/packages/service-library/requirements/_tools.txt index c12d45d2703..f487b7983b8 100644 --- a/packages/service-library/requirements/_tools.txt +++ b/packages/service-library/requirements/_tools.txt @@ -45,8 +45,10 @@ packaging==24.1 # build pathspec==0.12.1 # via black -pip==24.2 - # via pip-tools +pip==24.3.1 + # via + # -c requirements/_test.txt + # pip-tools pip-tools==7.4.1 # via -r requirements/../../../requirements/devenv.txt platformdirs==4.3.6 diff --git a/packages/service-library/src/servicelib/aiohttp/tracing.py b/packages/service-library/src/servicelib/aiohttp/tracing.py index 9947c12be0b..3da3b28e3b3 100644 --- a/packages/service-library/src/servicelib/aiohttp/tracing.py +++ b/packages/service-library/src/servicelib/aiohttp/tracing.py @@ -15,23 +15,39 @@ from opentelemetry.instrumentation.aiohttp_server import ( middleware as aiohttp_server_opentelemetry_middleware, # pylint:disable=no-name-in-module ) -from opentelemetry.instrumentation.aiopg import ( # pylint:disable=no-name-in-module - AiopgInstrumentor, -) -from opentelemetry.instrumentation.requests import RequestsInstrumentor from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from servicelib.logging_utils import log_context from settings_library.tracing import TracingSettings _logger = logging.getLogger(__name__) +try: + from opentelemetry.instrumentation.botocore import ( # type: ignore[import-not-found] + BotocoreInstrumentor, + ) + + HAS_BOTOCORE = True +except ImportError: + HAS_BOTOCORE = False +try: + from opentelemetry.instrumentation.aiopg import AiopgInstrumentor + + HAS_AIOPG = True +except ImportError: + HAS_AIOPG = False +try: + from opentelemetry.instrumentation.requests import RequestsInstrumentor + + HAS_REQUESTS = True +except ImportError: + HAS_REQUESTS = False def setup_tracing( app: web.Application, tracing_settings: TracingSettings, service_name: str, - instrument_aiopg: bool = False, # noqa: FBT001, FBT002 ) -> None: """ Sets up this service for a distributed tracing system (opentelemetry) @@ -91,6 +107,24 @@ def setup_tracing( # Instrument aiohttp client AioHttpClientInstrumentor().instrument() - if instrument_aiopg: - AiopgInstrumentor().instrument() - RequestsInstrumentor().instrument() + if HAS_AIOPG: + with log_context( + _logger, + logging.INFO, + msg="Attempting to add aio-pg opentelemetry autoinstrumentation...", + ): + AiopgInstrumentor().instrument() + if HAS_BOTOCORE: + with log_context( + _logger, + logging.INFO, + msg="Attempting to add botocore opentelemetry autoinstrumentation...", + ): + BotocoreInstrumentor().instrument() + if HAS_REQUESTS: + with log_context( + _logger, + logging.INFO, + msg="Attempting to add requests opentelemetry autoinstrumentation...", + ): + RequestsInstrumentor().instrument() diff --git a/packages/service-library/src/servicelib/fastapi/tracing.py b/packages/service-library/src/servicelib/fastapi/tracing.py index e0f670686f5..b5179a8a5f6 100644 --- a/packages/service-library/src/servicelib/fastapi/tracing.py +++ b/packages/service-library/src/servicelib/fastapi/tracing.py @@ -13,9 +13,49 @@ from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from servicelib.logging_utils import log_context from settings_library.tracing import TracingSettings -log = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) + +try: + from opentelemetry.instrumentation.asyncpg import ( # type: ignore[import-not-found] + AsyncPGInstrumentor, + ) + + HAS_ASYNCPG = True +except ImportError: + HAS_ASYNCPG = False + +try: + from opentelemetry.instrumentation.aiopg import AiopgInstrumentor + + HAS_AIOPG = True +except ImportError: + HAS_AIOPG = False + +try: + from opentelemetry.instrumentation.redis import RedisInstrumentor + + HAS_REDIS = True +except ImportError: + HAS_REDIS = False + +try: + from opentelemetry.instrumentation.botocore import ( # type: ignore[import-not-found] + BotocoreInstrumentor, + ) + + HAS_BOTOCORE = True +except ImportError: + HAS_BOTOCORE = False + +try: + from opentelemetry.instrumentation.requests import RequestsInstrumentor + + HAS_REQUESTS = True +except ImportError: + HAS_REQUESTS = False def setup_tracing( @@ -25,7 +65,7 @@ def setup_tracing( not tracing_settings.TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT and not tracing_settings.TRACING_OPENTELEMETRY_COLLECTOR_PORT ): - log.warning("Skipping opentelemetry tracing setup") + _logger.warning("Skipping opentelemetry tracing setup") return # Set up the tracer provider @@ -34,8 +74,8 @@ def setup_tracing( global_tracer_provider = trace.get_tracer_provider() assert isinstance(global_tracer_provider, TracerProvider) # nosec tracing_destination: str = f"{tracing_settings.TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT}:{tracing_settings.TRACING_OPENTELEMETRY_COLLECTOR_PORT}/v1/traces" - log.info( - "Trying to connect service %s to tracing collector at %s.", + _logger.info( + "Trying to connect service %s to opentelemetry tracing collector at %s.", service_name, tracing_destination, ) @@ -45,3 +85,39 @@ def setup_tracing( global_tracer_provider.add_span_processor(span_processor) # Instrument FastAPI FastAPIInstrumentor().instrument_app(app) + + if HAS_AIOPG: + with log_context( + _logger, + logging.INFO, + msg="Attempting to add asyncpg opentelemetry autoinstrumentation...", + ): + AiopgInstrumentor().instrument() + if HAS_ASYNCPG: + with log_context( + _logger, + logging.INFO, + msg="Attempting to add asyncpg opentelemetry autoinstrumentation...", + ): + AsyncPGInstrumentor().instrument() + if HAS_REDIS: + with log_context( + _logger, + logging.INFO, + msg="Attempting to add redis opentelemetry autoinstrumentation...", + ): + RedisInstrumentor().instrument() + if HAS_BOTOCORE: + with log_context( + _logger, + logging.INFO, + msg="Attempting to add botocore opentelemetry autoinstrumentation...", + ): + BotocoreInstrumentor().instrument() + if HAS_REQUESTS: + with log_context( + _logger, + logging.INFO, + msg="Attempting to add requests opentelemetry autoinstrumentation...", + ): + RequestsInstrumentor().instrument() diff --git a/packages/service-library/tests/aiohttp/test_tracing.py b/packages/service-library/tests/aiohttp/test_tracing.py index 2a1cc30eeac..389394f3d0d 100644 --- a/packages/service-library/tests/aiohttp/test_tracing.py +++ b/packages/service-library/tests/aiohttp/test_tracing.py @@ -2,8 +2,11 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable +import importlib from collections.abc import Callable +from typing import Any, Iterator +import pip import pytest from aiohttp import web from aiohttp.test_utils import TestClient @@ -42,6 +45,7 @@ async def test_valid_tracing_settings( aiohttp_client: Callable, set_and_clean_settings_env_vars: Callable, tracing_settings_in, + uninstrument_opentelemetry: Iterator[None], ) -> TestClient: app = web.Application() service_name = "simcore_service_webserver" @@ -66,6 +70,69 @@ async def test_invalid_tracing_settings( aiohttp_client: Callable, set_and_clean_settings_env_vars: Callable, tracing_settings_in, + uninstrument_opentelemetry: Iterator[None], ) -> TestClient: with pytest.raises(ValidationError): TracingSettings() + + +def install_package(package): + pip.main(["install", package]) + + +def uninstall_package(package): + pip.main(["uninstall", "-y", package]) + + +@pytest.fixture(scope="function") +def manage_package(request): + package, importname = request.param + install_package(package) + yield importname + uninstall_package(package) + + +@pytest.mark.parametrize( + "tracing_settings_in, manage_package", + [ + ( + ("http://opentelemetry-collector", 4318), + ( + "opentelemetry-instrumentation-botocore", + "opentelemetry.instrumentation.botocore", + ), + ), + ( + ("http://opentelemetry-collector", "4318"), + ( + "opentelemetry-instrumentation-aiopg", + "opentelemetry.instrumentation.aiopg", + ), + ), + ], + indirect=True, +) +async def test_tracing_setup_package_detection( + aiohttp_client: Callable, + set_and_clean_settings_env_vars: Callable[[], None], + tracing_settings_in: Callable[[], dict[str, Any]], + manage_package, + uninstrument_opentelemetry: Iterator[None], +): + package_name = manage_package + importlib.import_module(package_name) + # + app = web.Application() + service_name = "simcore_service_webserver" + tracing_settings = TracingSettings() + setup_tracing( + app, + service_name=service_name, + tracing_settings=tracing_settings, + ) + # idempotency + setup_tracing( + app, + service_name=service_name, + tracing_settings=tracing_settings, + ) diff --git a/packages/service-library/tests/conftest.py b/packages/service-library/tests/conftest.py index 712746ccce9..927ff75477f 100644 --- a/packages/service-library/tests/conftest.py +++ b/packages/service-library/tests/conftest.py @@ -99,3 +99,60 @@ async def _cleanup_redis_data(clients_manager: RedisClientsManager) -> None: await _cleanup_redis_data(clients_manager) yield _ await _cleanup_redis_data(clients_manager) + + +@pytest.fixture() +def uninstrument_opentelemetry(): + yield + try: + from opentelemetry.instrumentation.redis import RedisInstrumentor + + RedisInstrumentor().uninstrument() + except ImportError: + pass + try: + from opentelemetry.instrumentation.botocore import BotocoreInstrumentor + + BotocoreInstrumentor().uninstrument() + except ImportError: + pass + try: + from opentelemetry.instrumentation.requests import RequestsInstrumentor + + RequestsInstrumentor().uninstrument() + except ImportError: + pass + try: + from opentelemetry.instrumentation.aiopg import AiopgInstrumentor + + AiopgInstrumentor().uninstrument() + except ImportError: + pass + try: + from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor + + AsyncPGInstrumentor().uninstrument() + except ImportError: + pass + try: + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + + FastAPIInstrumentor().uninstrument() + except ImportError: + pass + try: + from opentelemetry.instrumentation.aiohttp_client import ( + AioHttpClientInstrumentor, + ) + + AioHttpClientInstrumentor().uninstrument() + except ImportError: + pass + try: + from opentelemetry.instrumentation.aiohttp_server import ( + AioHttpServerInstrumentor, + ) + + AioHttpServerInstrumentor().uninstrument() + except ImportError: + pass diff --git a/packages/service-library/tests/fastapi/test_tracing.py b/packages/service-library/tests/fastapi/test_tracing.py index 9364cb75a34..412b59b116d 100644 --- a/packages/service-library/tests/fastapi/test_tracing.py +++ b/packages/service-library/tests/fastapi/test_tracing.py @@ -1,11 +1,13 @@ # pylint: disable=all +import importlib import random import string from collections.abc import Callable -from typing import Any +from typing import Any, Iterator +import pip import pytest from fastapi import FastAPI from pydantic import ValidationError @@ -49,6 +51,7 @@ async def test_valid_tracing_settings( mocked_app: FastAPI, set_and_clean_settings_env_vars: Callable[[], None], tracing_settings_in: Callable[[], dict[str, Any]], + uninstrument_opentelemetry: Iterator[None], ): tracing_settings = TracingSettings() setup_tracing( @@ -84,6 +87,7 @@ async def test_invalid_tracing_settings( mocked_app: FastAPI, set_and_clean_settings_env_vars: Callable[[], None], tracing_settings_in: Callable[[], dict[str, Any]], + uninstrument_opentelemetry: Iterator[None], ): app = mocked_app with pytest.raises((BaseException, ValidationError, TypeError)): # noqa: PT012 @@ -93,3 +97,63 @@ async def test_invalid_tracing_settings( tracing_settings=tracing_settings, service_name="Mock-Openetlemetry-Pytest", ) + + +def install_package(package): + pip.main(["install", package]) + + +def uninstall_package(package): + pip.main(["uninstall", "-y", package]) + + +@pytest.fixture(scope="function") +def manage_package(request): + package, importname = request.param + install_package(package) + yield importname + uninstall_package(package) + + +@pytest.mark.parametrize( + "tracing_settings_in, manage_package", + [ + ( + ("http://opentelemetry-collector", 4318), + ( + "opentelemetry-instrumentation-botocore", + "opentelemetry.instrumentation.botocore", + ), + ), + ( + ("http://opentelemetry-collector", "4318"), + ( + "opentelemetry-instrumentation-aiopg", + "opentelemetry.instrumentation.aiopg", + ), + ), + ], + indirect=True, +) +async def test_tracing_setup_package_detection( + mocked_app: FastAPI, + set_and_clean_settings_env_vars: Callable[[], None], + tracing_settings_in: Callable[[], dict[str, Any]], + uninstrument_opentelemetry: Iterator[None], + manage_package, +): + package_name = manage_package + importlib.import_module(package_name) + # + tracing_settings = TracingSettings() + setup_tracing( + mocked_app, + tracing_settings=tracing_settings, + service_name="Mock-Openetlemetry-Pytest", + ) + # idempotency + setup_tracing( + mocked_app, + tracing_settings=tracing_settings, + service_name="Mock-Openetlemetry-Pytest", + ) diff --git a/packages/service-library/tests/rabbitmq/test_rabbitmq.py b/packages/service-library/tests/rabbitmq/test_rabbitmq.py index 56bcce4c026..5bc26c3be1e 100644 --- a/packages/service-library/tests/rabbitmq/test_rabbitmq.py +++ b/packages/service-library/tests/rabbitmq/test_rabbitmq.py @@ -266,7 +266,11 @@ async def _fail_once_then_succeed(message: Any) -> bool: message = entry.args[2] if message.headers == {}: original_message_count += 1 - if message.headers and message.headers["x-death"][0]["count"] == 1: + if ( + message.headers + and "x-death" in message.headers + and message.headers["x-death"][0]["count"] == 1 + ): requeued_message_count += 1 assert original_message_count == topics_count diff --git a/packages/simcore-sdk/requirements/_base.in b/packages/simcore-sdk/requirements/_base.in index a07a0b50b01..da34e87a026 100644 --- a/packages/simcore-sdk/requirements/_base.in +++ b/packages/simcore-sdk/requirements/_base.in @@ -12,6 +12,7 @@ aiocache aiofiles aiohttp aiopg[sa] +opentelemetry-instrumentation-aiopg packaging pint pydantic[email] diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 6a2a594c491..5eac02fa1ec 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -48,6 +48,7 @@ async-timeout==4.0.3 # via # aiopg # asyncpg + # redis asyncpg==0.29.0 # via sqlalchemy attrs==24.2.0 @@ -139,6 +140,10 @@ opentelemetry-api==1.27.0 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation + # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions @@ -153,7 +158,20 @@ opentelemetry-exporter-otlp-proto-grpc==1.27.0 opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp opentelemetry-instrumentation==0.48b0 - # via opentelemetry-instrumentation-requests + # via + # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests +opentelemetry-instrumentation-aiopg==0.48b0 + # via -r requirements/_base.in +opentelemetry-instrumentation-asyncpg==0.48b0 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-dbapi==0.48b0 + # via opentelemetry-instrumentation-aiopg +opentelemetry-instrumentation-redis==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-proto==1.27.0 @@ -168,6 +186,9 @@ opentelemetry-sdk==1.27.0 # opentelemetry-exporter-otlp-proto-http opentelemetry-semantic-conventions==0.48b0 # via + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.48b0 @@ -321,6 +342,9 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis yarl==1.12.1 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index 74afc082867..59f29515fe5 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -38,6 +38,8 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi +async-timeout==4.0.3 + # via redis attrs==24.2.0 # via # aiohttp @@ -141,6 +143,7 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions @@ -158,11 +161,14 @@ opentelemetry-instrumentation==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-redis==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-proto==1.27.0 @@ -179,6 +185,7 @@ opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.48b0 @@ -327,6 +334,7 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.12.1 # via # aio-pika diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index 51e2544cf88..92a441a0e25 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -74,6 +74,7 @@ async-timeout==4.0.3 # via # aiopg # asyncpg + # redis asyncpg==0.29.0 # via sqlalchemy attrs==23.2.0 @@ -270,61 +271,84 @@ multidict==6.0.5 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation + # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.48b0 # via + # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-aiopg==0.48b0 + # via -r requirements/../../../packages/simcore-sdk/requirements/_base.in +opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-asyncpg==0.48b0 + # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-dbapi==0.48b0 + # via opentelemetry-instrumentation-aiopg +opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-redis==0.48b0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.48b0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -620,6 +644,9 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis yarl==1.9.4 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index a58d343c4fe..0c7ff77b07f 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -65,6 +65,8 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi +async-timeout==4.0.3 + # via redis attrs==23.2.0 # via # aiohttp @@ -256,8 +258,11 @@ opentelemetry-api==1.26.0 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests + # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions opentelemetry-exporter-otlp==1.26.0 @@ -275,16 +280,26 @@ opentelemetry-exporter-otlp-proto-http==1.26.0 opentelemetry-instrumentation==0.47b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.47b0 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-botocore==0.47b0 + # via -r requirements/../../../packages/aws-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-redis==0.47b0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.47b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-propagator-aws-xray==1.0.1 + # via opentelemetry-instrumentation-botocore opentelemetry-proto==1.26.0 # via # opentelemetry-exporter-otlp-proto-common @@ -299,7 +314,9 @@ opentelemetry-sdk==1.26.0 opentelemetry-semantic-conventions==0.47b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.47b0 @@ -562,6 +579,7 @@ wrapt==1.16.0 # aiobotocore # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.9.4 # via # aio-pika diff --git a/services/autoscaling/requirements/_test.txt b/services/autoscaling/requirements/_test.txt index 47379c4d69f..8abc686eb76 100644 --- a/services/autoscaling/requirements/_test.txt +++ b/services/autoscaling/requirements/_test.txt @@ -6,6 +6,10 @@ anyio==4.3.0 # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in +async-timeout==4.0.3 + # via + # -c requirements/_base.txt + # redis attrs==23.2.0 # via # -c requirements/_base.txt diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index b705ea85b78..9876a3a1c20 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -226,6 +226,7 @@ def app_environment( "AUTOSCALING_EC2_SECRET_ACCESS_KEY": faker.pystr(), "AUTOSCALING_EC2_INSTANCES": "{}", "AUTOSCALING_SSM_ACCESS": "{}", + "AUTOSCALING_TRACING": "{}", "SSM_ACCESS_KEY_ID": faker.pystr(), "SSM_SECRET_ACCESS_KEY": faker.pystr(), "EC2_INSTANCES_KEY_NAME": faker.pystr(), @@ -366,6 +367,13 @@ def disabled_ec2(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch) monkeypatch.setenv("AUTOSCALING_EC2_ACCESS", "null") +@pytest.fixture +def disabled_opentelemetry( + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("AUTOSCALING_TRACING", "null") + + @pytest.fixture def disabled_ssm(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("AUTOSCALING_SSM_ACCESS", "null") diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py index 5811b43b2f0..372546149c1 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py @@ -79,6 +79,7 @@ def minimal_configuration( local_dask_scheduler_server_envs: EnvVarsDict, mocked_ec2_instances_envs: EnvVarsDict, disabled_rabbitmq: None, + disabled_opentelemetry: None, disable_dynamic_service_background_task: None, disable_buffers_pool_background_task: None, mocked_redis_server: None, diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index a1d4ec9de10..890adbe5508 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -41,7 +41,9 @@ arrow==1.3.0 asgiref==3.8.1 # via opentelemetry-instrumentation-asgi async-timeout==4.0.3 - # via asyncpg + # via + # asyncpg + # redis asyncpg==0.29.0 # via # -r requirements/_base.in @@ -180,55 +182,65 @@ multidict==6.0.5 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.27.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-asyncpg==0.48b0 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-redis==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -425,6 +437,7 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.9.4 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 38f29931595..9443ee269ef 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -63,6 +63,8 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi +async-timeout==4.0.3 + # via redis attrs==23.2.0 # via # aiohttp @@ -254,8 +256,11 @@ opentelemetry-api==1.26.0 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests + # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions opentelemetry-exporter-otlp==1.26.0 @@ -273,16 +278,26 @@ opentelemetry-exporter-otlp-proto-http==1.26.0 opentelemetry-instrumentation==0.47b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.47b0 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-botocore==0.47b0 + # via -r requirements/../../../packages/aws-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-redis==0.47b0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.47b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-propagator-aws-xray==1.0.1 + # via opentelemetry-instrumentation-botocore opentelemetry-proto==1.26.0 # via # opentelemetry-exporter-otlp-proto-common @@ -297,7 +312,9 @@ opentelemetry-sdk==1.26.0 opentelemetry-semantic-conventions==0.47b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.47b0 @@ -560,6 +577,7 @@ wrapt==1.16.0 # aiobotocore # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.9.4 # via # aio-pika diff --git a/services/clusters-keeper/requirements/_test.txt b/services/clusters-keeper/requirements/_test.txt index 00a7437644c..e2832a14944 100644 --- a/services/clusters-keeper/requirements/_test.txt +++ b/services/clusters-keeper/requirements/_test.txt @@ -19,6 +19,10 @@ anyio==4.3.0 # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in +async-timeout==4.0.3 + # via + # -c requirements/_base.txt + # redis attrs==23.2.0 # via # -c requirements/_base.txt diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt index 8cbd9e4a15a..6cdd686b12f 100644 --- a/services/dask-sidecar/requirements/_base.txt +++ b/services/dask-sidecar/requirements/_base.txt @@ -46,6 +46,8 @@ arrow==1.3.0 # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in +async-timeout==4.0.3 + # via redis attrs==23.2.0 # via # aiohttp @@ -187,6 +189,7 @@ opentelemetry-api==1.26.0 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions @@ -201,7 +204,11 @@ opentelemetry-exporter-otlp-proto-grpc==1.26.0 opentelemetry-exporter-otlp-proto-http==1.26.0 # via opentelemetry-exporter-otlp opentelemetry-instrumentation==0.47b0 - # via opentelemetry-instrumentation-requests + # via + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests +opentelemetry-instrumentation-redis==0.47b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-proto==1.26.0 @@ -216,6 +223,7 @@ opentelemetry-sdk==1.26.0 # opentelemetry-exporter-otlp-proto-http opentelemetry-semantic-conventions==0.47b0 # via + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.47b0 @@ -405,6 +413,7 @@ wrapt==1.16.0 # aiobotocore # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis xyzservices==2024.4.0 # via bokeh yarl==1.9.4 diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt index bf98d4efe31..f8fe44d6058 100644 --- a/services/datcore-adapter/requirements/_base.txt +++ b/services/datcore-adapter/requirements/_base.txt @@ -39,6 +39,8 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi +async-timeout==4.0.3 + # via redis attrs==23.2.0 # via # aiohttp @@ -164,6 +166,7 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions @@ -181,11 +184,14 @@ opentelemetry-instrumentation==0.47b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.47b0 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-fastapi==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-redis==0.47b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-proto==1.26.0 @@ -202,6 +208,7 @@ opentelemetry-semantic-conventions==0.47b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.47b0 @@ -367,6 +374,7 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.9.4 # via # aio-pika diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index 7e548a9fdcb..02162fe9a64 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -81,6 +81,7 @@ async-timeout==4.0.3 # via # aiopg # asyncpg + # redis asyncpg==0.29.0 # via sqlalchemy attrs==23.2.0 @@ -327,61 +328,84 @@ networkx==3.3 # via -r requirements/_base.in numpy==1.26.4 # via -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt -opentelemetry-api==1.26.0 +opentelemetry-api==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation + # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.48b0 # via + # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-aiopg==0.48b0 + # via -r requirements/../../../packages/simcore-sdk/requirements/_base.in +opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-asyncpg==0.48b0 + # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-dbapi==0.48b0 + # via opentelemetry-instrumentation-aiopg +opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-redis==0.48b0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.48b0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -745,6 +769,9 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis wsproto==1.2.0 # via simple-websocket yarl==1.9.4 diff --git a/services/director/requirements/_tools.txt b/services/director/requirements/_tools.txt index 38ed7220aed..24945ba6807 100644 --- a/services/director/requirements/_tools.txt +++ b/services/director/requirements/_tools.txt @@ -30,6 +30,8 @@ pathspec==0.9.0 # via black pep517==0.12.0 # via pip-tools +pip==24.3.1 + # via pip-tools pip-tools==6.4.0 # via -r requirements/_tools.in pyyaml==5.4 diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 30f09ba7c89..cb2cc603fb0 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -40,7 +40,9 @@ arrow==1.3.0 asgiref==3.8.1 # via opentelemetry-instrumentation-asgi async-timeout==4.0.3 - # via asyncpg + # via + # asyncpg + # redis asyncpg==0.29.0 # via sqlalchemy attrs==23.2.0 @@ -161,55 +163,65 @@ multidict==6.0.5 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.27.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-asyncpg==0.48b0 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-redis==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -394,6 +406,7 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis wsproto==1.2.0 # via simple-websocket yarl==1.9.4 diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index 66b45dc90c9..40c32b696ec 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -76,6 +76,7 @@ async-timeout==4.0.3 # via # aiopg # asyncpg + # redis asyncpg==0.29.0 # via sqlalchemy attrs==23.2.0 @@ -230,61 +231,84 @@ multidict==6.0.5 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation + # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.48b0 # via + # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-aiopg==0.48b0 + # via -r requirements/../../../packages/simcore-sdk/requirements/_base.in +opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-asyncpg==0.48b0 + # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-dbapi==0.48b0 + # via opentelemetry-instrumentation-aiopg +opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-redis==0.48b0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.48b0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -554,6 +578,9 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis wsproto==1.2.0 # via simple-websocket yarl==1.9.4 diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index 6669c45274b..26a626f01db 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -69,7 +69,9 @@ arrow==1.3.0 asgiref==3.8.1 # via opentelemetry-instrumentation-asgi async-timeout==4.0.3 - # via asyncpg + # via + # asyncpg + # redis asyncpg==0.29.0 # via sqlalchemy attrs==24.2.0 @@ -233,8 +235,12 @@ opentelemetry-api==1.27.0 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests + # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions opentelemetry-exporter-otlp==1.27.0 @@ -252,16 +258,29 @@ opentelemetry-exporter-otlp-proto-http==1.27.0 opentelemetry-instrumentation==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asyncpg==0.48b0 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-botocore==0.48b0 + # via -r requirements/../../../packages/aws-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-redis==0.48b0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.48b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-propagator-aws-xray==1.0.1 + # via opentelemetry-instrumentation-botocore opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common @@ -276,7 +295,10 @@ opentelemetry-sdk==1.27.0 opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.48b0 @@ -536,6 +558,7 @@ wrapt==1.16.0 # aiobotocore # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.15.4 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in diff --git a/services/efs-guardian/requirements/_test.txt b/services/efs-guardian/requirements/_test.txt index f188e8071de..8bdc1ec8ebf 100644 --- a/services/efs-guardian/requirements/_test.txt +++ b/services/efs-guardian/requirements/_test.txt @@ -23,6 +23,10 @@ anyio==4.6.2.post1 # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in +async-timeout==4.0.3 + # via + # -c requirements/_base.txt + # redis attrs==24.2.0 # via # -c requirements/_base.txt diff --git a/services/invitations/requirements/_base.txt b/services/invitations/requirements/_base.txt index 4d2a539cd68..c6e253b5e6a 100644 --- a/services/invitations/requirements/_base.txt +++ b/services/invitations/requirements/_base.txt @@ -35,6 +35,8 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi +async-timeout==4.0.3 + # via redis attrs==23.2.0 # via # aiohttp @@ -151,6 +153,7 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions @@ -168,11 +171,14 @@ opentelemetry-instrumentation==0.47b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.47b0 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-fastapi==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-redis==0.47b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-proto==1.26.0 @@ -189,6 +195,7 @@ opentelemetry-semantic-conventions==0.47b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.47b0 @@ -349,6 +356,7 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.9.4 # via # aio-pika diff --git a/services/opentelemetry-collector-config.yaml b/services/opentelemetry-collector-config.yaml index d3a1d09f605..14640ab61da 100644 --- a/services/opentelemetry-collector-config.yaml +++ b/services/opentelemetry-collector-config.yaml @@ -8,16 +8,12 @@ receivers: exporters: otlphttp: endpoint: ${TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT} # Adjust to your Jaeger endpoint - debug: - verbosity: detailed service: pipelines: traces: receivers: [otlp] - exporters: [otlphttp,debug] - telemetry: - logs: - level: "debug" + exporters: [otlphttp] + processors: [batch,probabilistic_sampler] processors: batch: timeout: 5s diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index fc510211449..88aae6375d4 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -43,7 +43,9 @@ arrow==1.3.0 asgiref==3.8.1 # via opentelemetry-instrumentation-asgi async-timeout==4.0.3 - # via asyncpg + # via + # asyncpg + # redis asyncpg==0.29.0 # via sqlalchemy attrs==23.2.0 @@ -190,55 +192,65 @@ multidict==6.0.5 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.27.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-asyncpg==0.48b0 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-redis==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -442,6 +454,7 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis wsproto==1.2.0 # via simple-websocket yarl==1.9.4 diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index ffc209bb6df..97a3bd129b7 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -69,7 +69,9 @@ arrow==1.3.0 asgiref==3.8.1 # via opentelemetry-instrumentation-asgi async-timeout==4.0.3 - # via asyncpg + # via + # asyncpg + # redis asyncpg==0.29.0 # via sqlalchemy attrs==23.2.0 @@ -255,8 +257,12 @@ opentelemetry-api==1.26.0 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests + # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions opentelemetry-exporter-otlp==1.26.0 @@ -274,16 +280,29 @@ opentelemetry-exporter-otlp-proto-http==1.26.0 opentelemetry-instrumentation==0.47b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.47b0 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asyncpg==0.47b0 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-botocore==0.47b0 + # via -r requirements/../../../packages/aws-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-redis==0.47b0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.47b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-propagator-aws-xray==1.0.1 + # via opentelemetry-instrumentation-botocore opentelemetry-proto==1.26.0 # via # opentelemetry-exporter-otlp-proto-common @@ -298,7 +317,10 @@ opentelemetry-sdk==1.26.0 opentelemetry-semantic-conventions==0.47b0 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.47b0 @@ -597,6 +619,7 @@ wrapt==1.16.0 # aiobotocore # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.9.4 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in diff --git a/services/resource-usage-tracker/requirements/_test.txt b/services/resource-usage-tracker/requirements/_test.txt index 4db08363ded..e70753feb19 100644 --- a/services/resource-usage-tracker/requirements/_test.txt +++ b/services/resource-usage-tracker/requirements/_test.txt @@ -10,6 +10,10 @@ anyio==4.3.0 # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in +async-timeout==4.0.3 + # via + # -c requirements/_base.txt + # redis attrs==23.2.0 # via # -c requirements/_base.txt diff --git a/services/storage/requirements/_base.in b/services/storage/requirements/_base.in index 2db40016724..fcebd5ba20a 100644 --- a/services/storage/requirements/_base.in +++ b/services/storage/requirements/_base.in @@ -19,6 +19,7 @@ aiofiles # i/o aiohttp # server aiohttp-swagger[performance] # server aiopg[sa] # db +opentelemetry-instrumentation-botocore packaging tenacity typer diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index d5a191d6e58..edadd851b65 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -78,6 +78,7 @@ async-timeout==4.0.3 # via # aiopg # asyncpg + # redis asyncpg==0.29.0 # via sqlalchemy attrs==23.2.0 @@ -242,8 +243,12 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests + # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions opentelemetry-exporter-otlp==1.26.0 @@ -263,7 +268,10 @@ opentelemetry-instrumentation==0.47b0 # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aiohttp-client==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_aiohttp.in @@ -271,12 +279,24 @@ opentelemetry-instrumentation-aiohttp-server==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_aiohttp.in opentelemetry-instrumentation-aiopg==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_aiohttp.in +opentelemetry-instrumentation-asyncpg==0.47b0 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-botocore==0.47b0 + # via + # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/_base.in opentelemetry-instrumentation-dbapi==0.47b0 # via opentelemetry-instrumentation-aiopg +opentelemetry-instrumentation-redis==0.47b0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.47b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-propagator-aws-xray==1.0.1 + # via opentelemetry-instrumentation-botocore opentelemetry-proto==1.26.0 # via # opentelemetry-exporter-otlp-proto-common @@ -292,7 +312,10 @@ opentelemetry-semantic-conventions==0.47b0 # via # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server + # opentelemetry-instrumentation-asyncpg + # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk opentelemetry-util-http==0.47b0 @@ -570,6 +593,7 @@ wrapt==1.16.0 # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis yarl==1.9.4 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index f0132fe4c7c..1e33824a7c0 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -13,6 +13,10 @@ aiosignal==1.3.1 # aiohttp antlr4-python3-runtime==4.13.2 # via moto +async-timeout==4.0.3 + # via + # -c requirements/_base.txt + # redis attrs==23.2.0 # via # -c requirements/_base.txt diff --git a/services/storage/src/simcore_service_storage/application.py b/services/storage/src/simcore_service_storage/application.py index 7ad35793bd7..f3a396cd3d4 100644 --- a/services/storage/src/simcore_service_storage/application.py +++ b/services/storage/src/simcore_service_storage/application.py @@ -56,7 +56,6 @@ def create(settings: Settings) -> web.Application: app, tracing_settings=tracing_settings, service_name=APP_NAME, - instrument_aiopg=True, ) setup_db(app) diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index d566b8d9112..5b42c95fffd 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -89,6 +89,7 @@ async-timeout==4.0.3 # via # aiohttp # aiopg + # redis asyncpg==0.27.0 # via # -r requirements/_base.in @@ -269,7 +270,7 @@ openapi-spec-validator==0.4.0 # via openapi-core openpyxl==3.0.9 # via -r requirements/_base.in -opentelemetry-api==1.26.0 +opentelemetry-api==1.27.0 # via # -r requirements/../../../../packages/service-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -279,62 +280,77 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.27.0 # via # -r requirements/../../../../packages/service-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.48b0 # via # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-aiohttp-client==0.47b0 +opentelemetry-instrumentation-aiohttp-client==0.48b0 # via -r requirements/../../../../packages/service-library/requirements/_aiohttp.in -opentelemetry-instrumentation-aiohttp-server==0.47b0 +opentelemetry-instrumentation-aiohttp-server==0.48b0 # via -r requirements/../../../../packages/service-library/requirements/_aiohttp.in -opentelemetry-instrumentation-aiopg==0.47b0 +opentelemetry-instrumentation-aiopg==0.48b0 # via # -r requirements/../../../../packages/service-library/requirements/_aiohttp.in + # -r requirements/../../../../packages/simcore-sdk/requirements/_base.in # -r requirements/_base.in -opentelemetry-instrumentation-dbapi==0.47b0 +opentelemetry-instrumentation-asyncpg==0.48b0 + # via + # -r requirements/../../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in +opentelemetry-instrumentation-dbapi==0.48b0 # via opentelemetry-instrumentation-aiopg -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-redis==0.48b0 + # via + # -r requirements/../../../../packages/service-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.48b0 # via # -r requirements/../../../../packages/service-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.27.0 # via # -r requirements/../../../../packages/service-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.48b0 # via # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server @@ -619,6 +635,7 @@ wrapt==1.16.0 # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-redis yarl==1.9.4 # via # -c requirements/./constraints.txt diff --git a/services/web/server/requirements/_test.txt b/services/web/server/requirements/_test.txt index 3aab7cde47d..67fcd247fda 100644 --- a/services/web/server/requirements/_test.txt +++ b/services/web/server/requirements/_test.txt @@ -18,6 +18,7 @@ async-timeout==4.0.3 # via # -c requirements/_base.txt # aiohttp + # redis asyncpg==0.27.0 # via # -c requirements/_base.txt diff --git a/services/web/server/src/simcore_service_webserver/tracing.py b/services/web/server/src/simcore_service_webserver/tracing.py index 5eafffcaa37..23041d95238 100644 --- a/services/web/server/src/simcore_service_webserver/tracing.py +++ b/services/web/server/src/simcore_service_webserver/tracing.py @@ -26,6 +26,5 @@ def setup_app_tracing(app: web.Application): setup_tracing( app, tracing_settings=tracing_settings, - instrument_aiopg=True, service_name=APP_NAME, ) From 43f678f26f88076f45e239d1352cdcc53fa88813 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:21:43 +0100 Subject: [PATCH 10/22] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Using=20iso=20timede?= =?UTF-8?q?lta=20in=20settings=20(#6656)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .env-devel | 4 ++-- services/agent/src/simcore_service_agent/core/settings.py | 4 ++-- .../src/simcore_service_agent/services/volumes_manager.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env-devel b/.env-devel index 02f2a9d939e..978062f428b 100644 --- a/.env-devel +++ b/.env-devel @@ -88,7 +88,7 @@ COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=PRESIGNED COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=PRESIGNED DIRECTOR_V2_DEV_FEATURES_ENABLED=0 DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED=1 -DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL=0 +DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL=PT0S DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}' DIRECTOR_V2_HOST=director-v2 DIRECTOR_V2_LOGLEVEL=INFO @@ -108,7 +108,7 @@ DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=3600 # DIRECTOR_V2 ---- DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG DYNAMIC_SCHEDULER_PROFILING=1 -DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=3600 +DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=PT1H FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "unknown@osparc.io", "affiliation": "unknown"}}' diff --git a/services/agent/src/simcore_service_agent/core/settings.py b/services/agent/src/simcore_service_agent/core/settings.py index d901a34ca90..756bf2cac28 100644 --- a/services/agent/src/simcore_service_agent/core/settings.py +++ b/services/agent/src/simcore_service_agent/core/settings.py @@ -53,14 +53,14 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): AGENT_VOLUMES_CLEANUP_INTERVAL: timedelta = Field( timedelta(minutes=1), description="interval for running volumes removal" ) - AGENT_VOLUMES_CLENUP_BOOK_KEEPING_INTERVAL: timedelta = Field( + AGENT_VOLUMES_CLEANUP_BOOK_KEEPING_INTERVAL: timedelta = Field( timedelta(minutes=1), description=( "interval at which to scan for unsued volumes and keep track since " "they were detected as being unused" ), ) - AGENT_VOLUMES_CLENUP_REMOVE_VOLUMES_INACTIVE_FOR: timedelta = Field( + AGENT_VOLUMES_CLEANUP_REMOVE_VOLUMES_INACTIVE_FOR: timedelta = Field( timedelta(minutes=65), description=( "if a volume is unused for more than this interval it can be removed. " diff --git a/services/agent/src/simcore_service_agent/services/volumes_manager.py b/services/agent/src/simcore_service_agent/services/volumes_manager.py index 526589a2c9c..fa5a0cd1b17 100644 --- a/services/agent/src/simcore_service_agent/services/volumes_manager.py +++ b/services/agent/src/simcore_service_agent/services/volumes_manager.py @@ -174,9 +174,9 @@ async def _on_startup() -> None: volumes_manager = VolumesManager( app=app, - book_keeping_interval=settings.AGENT_VOLUMES_CLENUP_BOOK_KEEPING_INTERVAL, + book_keeping_interval=settings.AGENT_VOLUMES_CLEANUP_BOOK_KEEPING_INTERVAL, volume_cleanup_interval=settings.AGENT_VOLUMES_CLEANUP_INTERVAL, - remove_volumes_inactive_for=settings.AGENT_VOLUMES_CLENUP_REMOVE_VOLUMES_INACTIVE_FOR.total_seconds(), + remove_volumes_inactive_for=settings.AGENT_VOLUMES_CLEANUP_REMOVE_VOLUMES_INACTIVE_FOR.total_seconds(), ) volumes_manager.set_to_app_state(app) await volumes_manager.setup() From e1354eaf8925a59eb2fa4f97b926ca62bff8555d Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:28:17 +0100 Subject: [PATCH 11/22] =?UTF-8?q?=F0=9F=94=A8Cluster=20monitoring=20script?= =?UTF-8?q?:=20update=20due=20to=20latest=20changes=20in=20osparc-config?= =?UTF-8?q?=20(#6672)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../autoscaled_monitor/cli.py | 13 +++++-------- .../autoscaled_monitor/constants.py | 1 + 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py b/scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py index 189dde5c0df..20e1d380d12 100644 --- a/scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py +++ b/scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py @@ -13,6 +13,7 @@ DEFAULT_COMPUTATIONAL_EC2_FORMAT_WORKERS, DEFAULT_DYNAMIC_EC2_FORMAT, DEPLOY_SSH_KEY_PARSER, + UNIFIED_SSH_KEY_PARSE, wallet_id_spec, ) from .ec2 import autoscaling_ec2_client, cluster_keeper_ec2_client @@ -78,16 +79,12 @@ def main( # locate ssh key path for file_path in deploy_config.glob("**/*.pem"): - if any(_ in file_path.name for _ in ["license", "pkcs8"]): - continue - # very bad HACK where the license file contain openssh in the name - if ( - any(_ in f"{file_path}" for _ in ("sim4life.io", "osparc-master")) - and "openssh" not in f"{file_path}" - ): + if any(_ in file_path.name for _ in ["license", "pkcs8", "dask"]): continue - if DEPLOY_SSH_KEY_PARSER.parse(f"{file_path.name}") is not None: + if DEPLOY_SSH_KEY_PARSER.parse( + f"{file_path.name}" + ) is not None or UNIFIED_SSH_KEY_PARSE.parse(f"{file_path.name}"): rich.print( f"will be using following ssh_key_path: {file_path}. " "TIP: if wrong adapt the code or manually remove some of them." diff --git a/scripts/maintenance/computational-clusters/autoscaled_monitor/constants.py b/scripts/maintenance/computational-clusters/autoscaled_monitor/constants.py index 661039bed39..6b8b1038205 100644 --- a/scripts/maintenance/computational-clusters/autoscaled_monitor/constants.py +++ b/scripts/maintenance/computational-clusters/autoscaled_monitor/constants.py @@ -22,6 +22,7 @@ def wallet_id_spec(text) -> None | int: DEPLOY_SSH_KEY_PARSER: Final[parse.Parser] = parse.compile( r"{prefix}-{random_name}.pem" ) +UNIFIED_SSH_KEY_PARSE: Final[parse.Parser] = parse.compile("sshkey.pem") MINUTE: Final[int] = 60 HOUR: Final[int] = 60 * MINUTE From 2f0cfb0d70265af83232ef05429aa057e426efab Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Tue, 5 Nov 2024 17:03:50 +0100 Subject: [PATCH 12/22] =?UTF-8?q?=F0=9F=8E=A8=20[Frontend]=20Highlight=20w?= =?UTF-8?q?orkbench=20elements=20(#6670)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../source/class/osparc/file/FileDrop.js | 6 +++-- .../source/class/osparc/file/FilePicker.js | 4 +-- .../class/osparc/form/renderer/PropForm.js | 12 +++++++++ .../source/class/osparc/info/StudyMedium.js | 4 ++- .../class/osparc/metadata/ServicesInStudy.js | 4 ++- .../osparc/navigation/StudyTitleWOptions.js | 4 ++- .../osparc/node/slideshow/BaseNodeView.js | 4 ++- .../class/osparc/service/ServiceListItem.js | 4 ++- .../source/class/osparc/theme/Appearance.js | 18 ++++++------- .../source/class/osparc/theme/ColorDark.js | 8 +++--- .../source/class/osparc/theme/ColorLight.js | 8 +++--- .../source/class/osparc/theme/Decoration.js | 4 +-- .../source/class/osparc/widget/NodesTree.js | 8 ++++-- .../source/class/osparc/widget/Renamer.js | 1 - .../class/osparc/widget/StudyTitleOnlyTree.js | 4 ++- .../class/osparc/workbench/BaseNodeUI.js | 10 +++---- .../source/class/osparc/workbench/EdgeUI.js | 9 +++++++ .../class/osparc/workbench/ServiceCatalog.js | 4 ++- .../class/osparc/workbench/WorkbenchUI.js | 27 ++++++++++++++++--- 19 files changed, 98 insertions(+), 45 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/file/FileDrop.js b/services/static-webserver/client/source/class/osparc/file/FileDrop.js index f7aa0734b2f..106a6c66b51 100644 --- a/services/static-webserver/client/source/class/osparc/file/FileDrop.js +++ b/services/static-webserver/client/source/class/osparc/file/FileDrop.js @@ -102,7 +102,9 @@ qx.Class.define("osparc.file.FileDrop", { } } return files; - } + }, + + ONE_FILE_ONLY: qx.locale.Manager.tr("Only one file at a time is accepted.") + "
" + qx.locale.Manager.tr("Please zip all files together."), }, events: { @@ -299,7 +301,7 @@ qx.Class.define("osparc.file.FileDrop", { pos: this.__pointerFileEventToScreenPos(e) }); } else { - osparc.FlashMessenger.getInstance().logAs(this.tr("Only one file at a time is accepted."), "ERROR"); + osparc.FlashMessenger.getInstance().logAs(osparc.file.FileDrop.ONE_FILE_ONLY, "ERROR"); } } else { osparc.FlashMessenger.getInstance().logAs(this.tr("Folders are not accepted. You might want to upload a zip file."), "ERROR"); diff --git a/services/static-webserver/client/source/class/osparc/file/FilePicker.js b/services/static-webserver/client/source/class/osparc/file/FilePicker.js index 9b2b6872512..67d67c8455a 100644 --- a/services/static-webserver/client/source/class/osparc/file/FilePicker.js +++ b/services/static-webserver/client/source/class/osparc/file/FilePicker.js @@ -417,7 +417,7 @@ qx.Class.define("osparc.file.FilePicker", { }, uploadPendingFiles: function(files) { - if (files.length > 0) { + if (files.length) { if (files.length === 1) { const fileUploader = new osparc.file.FileUploader(this.getNode()); fileUploader.addListener("uploadAborted", () => this.__resetOutput()); @@ -428,7 +428,7 @@ qx.Class.define("osparc.file.FilePicker", { fileUploader.retrieveUrlAndUpload(files[0]); return true; } - osparc.FlashMessenger.getInstance().logAs(this.tr("Only one file is accepted"), "ERROR"); + osparc.FlashMessenger.getInstance().logAs(osparc.file.FileDrop.ONE_FILE_ONLY, "ERROR"); } return false; }, diff --git a/services/static-webserver/client/source/class/osparc/form/renderer/PropForm.js b/services/static-webserver/client/source/class/osparc/form/renderer/PropForm.js index ac609258130..ccd205a92e0 100644 --- a/services/static-webserver/client/source/class/osparc/form/renderer/PropForm.js +++ b/services/static-webserver/client/source/class/osparc/form/renderer/PropForm.js @@ -36,6 +36,7 @@ qx.Class.define("osparc.form.renderer.PropForm", { }, events: { + "highlightEdge": "qx.event.type.Data", "linkFieldModified": "qx.event.type.Data", "fileRequested": "qx.event.type.Data", "filePickerRequested": "qx.event.type.Data", @@ -928,6 +929,17 @@ qx.Class.define("osparc.form.renderer.PropForm", { nodeUuid: fromNodeId, output: fromPortId }; + const highlightEdgeUI = highlight => { + this.fireDataEvent("highlightEdge", { + highlight, + toNodeId: this.getNode().getNodeId(), + toPortId, + fromNodeId, + fromPortId, + }); + }; + ctrlLink.addListener("mouseover", () => highlightEdgeUI(true)); + ctrlLink.addListener("mouseout", () => highlightEdgeUI(false)); const workbench = study.getWorkbench(); const fromNode = workbench.getNode(fromNodeId); diff --git a/services/static-webserver/client/source/class/osparc/info/StudyMedium.js b/services/static-webserver/client/source/class/osparc/info/StudyMedium.js index e8c77c184dc..cb9eab2b432 100644 --- a/services/static-webserver/client/source/class/osparc/info/StudyMedium.js +++ b/services/static-webserver/client/source/class/osparc/info/StudyMedium.js @@ -162,7 +162,9 @@ qx.Class.define("osparc.info.StudyMedium", { const title = this.tr("Study Information"); const width = osparc.info.CardLarge.WIDTH; const height = osparc.info.CardLarge.HEIGHT; - osparc.ui.window.Window.popUpInWindow(studyDetails, title, width, height); + osparc.ui.window.Window.popUpInWindow(studyDetails, title, width, height).set({ + maxHeight: height + }); } } }); diff --git a/services/static-webserver/client/source/class/osparc/metadata/ServicesInStudy.js b/services/static-webserver/client/source/class/osparc/metadata/ServicesInStudy.js index 9f1dbe17600..b51dc1c7515 100644 --- a/services/static-webserver/client/source/class/osparc/metadata/ServicesInStudy.js +++ b/services/static-webserver/client/source/class/osparc/metadata/ServicesInStudy.js @@ -150,7 +150,9 @@ qx.Class.define("osparc.metadata.ServicesInStudy", { const title = this.tr("Service information"); const width = osparc.info.CardLarge.WIDTH; const height = osparc.info.CardLarge.HEIGHT; - osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height); + osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height).set({ + maxHeight: height + }); }, this); this._servicesGrid.add(infoButton, { row: i, diff --git a/services/static-webserver/client/source/class/osparc/navigation/StudyTitleWOptions.js b/services/static-webserver/client/source/class/osparc/navigation/StudyTitleWOptions.js index f580df87fcd..342f0de8cee 100644 --- a/services/static-webserver/client/source/class/osparc/navigation/StudyTitleWOptions.js +++ b/services/static-webserver/client/source/class/osparc/navigation/StudyTitleWOptions.js @@ -63,7 +63,9 @@ qx.Class.define("osparc.navigation.StudyTitleWOptions", { const title = this.tr("Information"); const width = osparc.info.CardLarge.WIDTH; const height = osparc.info.CardLarge.HEIGHT; - osparc.ui.window.Window.popUpInWindow(widget, title, width, height); + osparc.ui.window.Window.popUpInWindow(widget, title, width, height).set({ + maxHeight: height + }); }); break; case "study-menu-download-logs": diff --git a/services/static-webserver/client/source/class/osparc/node/slideshow/BaseNodeView.js b/services/static-webserver/client/source/class/osparc/node/slideshow/BaseNodeView.js index 84acc271802..e7de026cd94 100644 --- a/services/static-webserver/client/source/class/osparc/node/slideshow/BaseNodeView.js +++ b/services/static-webserver/client/source/class/osparc/node/slideshow/BaseNodeView.js @@ -225,7 +225,9 @@ qx.Class.define("osparc.node.slideshow.BaseNodeView", { const title = this.tr("Service information"); const width = osparc.info.CardLarge.WIDTH; const height = osparc.info.CardLarge.HEIGHT; - osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height); + osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height).set({ + maxHeight: height + }); }, __openInstructions: function() { diff --git a/services/static-webserver/client/source/class/osparc/service/ServiceListItem.js b/services/static-webserver/client/source/class/osparc/service/ServiceListItem.js index e93cf802333..959859389ac 100644 --- a/services/static-webserver/client/source/class/osparc/service/ServiceListItem.js +++ b/services/static-webserver/client/source/class/osparc/service/ServiceListItem.js @@ -164,7 +164,9 @@ qx.Class.define("osparc.service.ServiceListItem", { const title = this.tr("Service information"); const width = osparc.info.CardLarge.WIDTH; const height = osparc.info.CardLarge.HEIGHT; - osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height); + osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height).set({ + maxHeight: height + }); }); }, diff --git a/services/static-webserver/client/source/class/osparc/theme/Appearance.js b/services/static-webserver/client/source/class/osparc/theme/Appearance.js index 324cf65ff48..16facaa2949 100644 --- a/services/static-webserver/client/source/class/osparc/theme/Appearance.js +++ b/services/static-webserver/client/source/class/osparc/theme/Appearance.js @@ -192,19 +192,19 @@ qx.Theme.define("osparc.theme.Appearance", { WINDOW-SMALL-CAP CHOOSER --------------------------------------------------------------------------- */ - "window-small-cap": { + "node-ui-cap": { include: "window", // get all the settings from window alias: "window", // redirect kids to window/kid style: function(states) { return { backgroundColor: states.selected ? "node-selected-background" : "node-background", textColor: states.selected ? "text-selected" : "text", - decorator: states.maximized ? "window-small-cap-maximized" : "window-small-cap" + decorator: states.maximized ? "node-ui-cap-maximized" : "node-ui-cap" }; } }, - "window-small-cap/captionbar": { + "node-ui-cap/captionbar": { include: "window/captionbar", // load defaults from window captionbar alias: "window/captionbar", // redirect kids style: function(states) { @@ -217,7 +217,7 @@ qx.Theme.define("osparc.theme.Appearance", { } }, - "window-small-cap/title": { + "node-ui-cap/title": { include: "window/title", style: function(states) { return { @@ -228,7 +228,7 @@ qx.Theme.define("osparc.theme.Appearance", { } }, - "window-small-cap/minimize-button": { + "node-ui-cap/minimize-button": { alias: "window/minimize-button", include: "window/minimize-button", style: function(states) { @@ -238,7 +238,7 @@ qx.Theme.define("osparc.theme.Appearance", { } }, - "window-small-cap/restore-button": { + "node-ui-cap/restore-button": { alias: "window/restore-button", include: "window/restore-button", style: function(states) { @@ -248,7 +248,7 @@ qx.Theme.define("osparc.theme.Appearance", { } }, - "window-small-cap/maximize-button": { + "node-ui-cap/maximize-button": { alias: "window/maximize-button", include: "window/maximize-button", style: function(states) { @@ -258,7 +258,7 @@ qx.Theme.define("osparc.theme.Appearance", { } }, - "window-small-cap/close-button": { + "node-ui-cap/close-button": { alias: "window/close-button", include: "window/close-button", style: function(states) { @@ -268,7 +268,7 @@ qx.Theme.define("osparc.theme.Appearance", { } }, - "window-small-cap/progress": "progressbar", + "node-ui-cap/progress": "progressbar", "service-window": { include: "window", diff --git a/services/static-webserver/client/source/class/osparc/theme/ColorDark.js b/services/static-webserver/client/source/class/osparc/theme/ColorDark.js index 0d4085c3153..ca275a2371d 100644 --- a/services/static-webserver/client/source/class/osparc/theme/ColorDark.js +++ b/services/static-webserver/client/source/class/osparc/theme/ColorDark.js @@ -68,6 +68,8 @@ qx.Theme.define("osparc.theme.ColorDark", { "window-caption-background-active": "background-main-3", "window-caption-text": "text", "window-caption-text-active": "c12", + "window-border": "background-main-2", + "window-border-inner": "background-main-1", // material-button "material-button-background": "fab-background", @@ -111,10 +113,6 @@ qx.Theme.define("osparc.theme.ColorDark", { "button-box-pressed": "background-main-4", "border-lead": "c07", - // window - "window-border": "background-main-2", - "window-border-inner": "background-main-1", - // group box "white-box-border": "background-main-2", @@ -172,7 +170,7 @@ qx.Theme.define("osparc.theme.ColorDark", { "workbench-start-hint": "#505050", "node-background": "rgba(113, 157, 181, 0.5)", - "node-selected-background": "background-selected", + "node-selected-background": "strong-main", "node-title-text": "text-selected", "node-port-text": "#454545", diff --git a/services/static-webserver/client/source/class/osparc/theme/ColorLight.js b/services/static-webserver/client/source/class/osparc/theme/ColorLight.js index 6feedc258b6..54f1e83d0ea 100644 --- a/services/static-webserver/client/source/class/osparc/theme/ColorLight.js +++ b/services/static-webserver/client/source/class/osparc/theme/ColorLight.js @@ -68,6 +68,8 @@ qx.Theme.define("osparc.theme.ColorLight", { "window-caption-background-active": "background-main-3", "window-caption-text": "text", "window-caption-text-active": "c12", + "window-border": "background-main-2", + "window-border-inner": "background-main-1", // material-button "material-button-background": "fab-background", @@ -111,10 +113,6 @@ qx.Theme.define("osparc.theme.ColorLight", { "button-box-pressed": "background-main-4", "border-lead": "c07", - // window - "window-border": "background-main-2", - "window-border-inner": "background-main-1", - // group box "white-box-border": "background-main-2", @@ -172,7 +170,7 @@ qx.Theme.define("osparc.theme.ColorLight", { "workbench-start-hint": "#AFAFAF", "node-background": "rgba(113, 157, 181, 0.35)", - "node-selected-background": "background-selected", + "node-selected-background": "strong-main", "node-title-text": "#232323", "node-port-text": "#454545", diff --git a/services/static-webserver/client/source/class/osparc/theme/Decoration.js b/services/static-webserver/client/source/class/osparc/theme/Decoration.js index 5ec725e45f4..7eae08b8f89 100644 --- a/services/static-webserver/client/source/class/osparc/theme/Decoration.js +++ b/services/static-webserver/client/source/class/osparc/theme/Decoration.js @@ -137,7 +137,7 @@ qx.Theme.define("osparc.theme.Decoration", { } }, - "window-small-cap": { + "node-ui-cap": { include: "service-window", style: { shadowBlurRadius: 0, @@ -150,7 +150,7 @@ qx.Theme.define("osparc.theme.Decoration", { } }, - "window-small-cap-maximized": { + "node-ui-cap-maximized": { include: "service-window-maximized", style: { width: 1, diff --git a/services/static-webserver/client/source/class/osparc/widget/NodesTree.js b/services/static-webserver/client/source/class/osparc/widget/NodesTree.js index e35409e5b34..0b543b6a158 100644 --- a/services/static-webserver/client/source/class/osparc/widget/NodesTree.js +++ b/services/static-webserver/client/source/class/osparc/widget/NodesTree.js @@ -285,7 +285,9 @@ qx.Class.define("osparc.widget.NodesTree", { if (nodeId === study.getUuid()) { const studyDetails = new osparc.info.StudyLarge(study); const title = this.tr("Study Information"); - osparc.ui.window.Window.popUpInWindow(studyDetails, title, width, height); + osparc.ui.window.Window.popUpInWindow(studyDetails, title, width, height).set({ + maxHeight: height + }); } else { const node = study.getWorkbench().getNode(nodeId); const serviceDetails = new osparc.info.ServiceLarge(node.getMetaData(), { @@ -294,7 +296,9 @@ qx.Class.define("osparc.widget.NodesTree", { studyId: study.getUuid() }); const title = this.tr("Service information"); - osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height); + osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height).set({ + maxHeight: height + }); } } }, diff --git a/services/static-webserver/client/source/class/osparc/widget/Renamer.js b/services/static-webserver/client/source/class/osparc/widget/Renamer.js index 8b7dcc161f4..fbba9f5184e 100644 --- a/services/static-webserver/client/source/class/osparc/widget/Renamer.js +++ b/services/static-webserver/client/source/class/osparc/widget/Renamer.js @@ -44,7 +44,6 @@ qx.Class.define("osparc.widget.Renamer", { const minWidth = 150; const labelWidth = oldLabel ? Math.min(Math.max(parseInt(oldLabel.length*4), minWidth), maxWidth) : minWidth; this.set({ - appearance: "window-small-cap", layout: new qx.ui.layout.VBox(5), autoDestroy: true, padding: 2, diff --git a/services/static-webserver/client/source/class/osparc/widget/StudyTitleOnlyTree.js b/services/static-webserver/client/source/class/osparc/widget/StudyTitleOnlyTree.js index a153891b8cb..6d3499b2364 100644 --- a/services/static-webserver/client/source/class/osparc/widget/StudyTitleOnlyTree.js +++ b/services/static-webserver/client/source/class/osparc/widget/StudyTitleOnlyTree.js @@ -48,7 +48,9 @@ qx.Class.define("osparc.widget.StudyTitleOnlyTree", { const title = this.tr("Study Information"); const width = osparc.info.CardLarge.WIDTH; const height = osparc.info.CardLarge.HEIGHT; - osparc.ui.window.Window.popUpInWindow(studyDetails, title, width, height); + osparc.ui.window.Window.popUpInWindow(studyDetails, title, width, height).set({ + maxHeight: height + }); }, selectStudyItem: function() { diff --git a/services/static-webserver/client/source/class/osparc/workbench/BaseNodeUI.js b/services/static-webserver/client/source/class/osparc/workbench/BaseNodeUI.js index 375b0c7f761..bd43406a518 100644 --- a/services/static-webserver/client/source/class/osparc/workbench/BaseNodeUI.js +++ b/services/static-webserver/client/source/class/osparc/workbench/BaseNodeUI.js @@ -28,6 +28,7 @@ qx.Class.define("osparc.workbench.BaseNodeUI", { grid.setColumnFlex(1, 1); this.set({ + appearance: "node-ui-cap", layout: grid, showMinimize: false, showMaximize: false, @@ -84,11 +85,6 @@ qx.Class.define("osparc.workbench.BaseNodeUI", { nullable: false }, - appearance: { - init: "window-small-cap", - refine: true - }, - isMovable: { check: "Boolean", init: true, @@ -113,8 +109,8 @@ qx.Class.define("osparc.workbench.BaseNodeUI", { }, captionHeight: function() { - return osparc.theme.Appearance.appearances["window-small-cap/captionbar"].style().height || - osparc.theme.Appearance.appearances["window-small-cap/captionbar"].style().minHeight; + return osparc.theme.Appearance.appearances["node-ui-cap/captionbar"].style().height || + osparc.theme.Appearance.appearances["node-ui-cap/captionbar"].style().minHeight; } }, diff --git a/services/static-webserver/client/source/class/osparc/workbench/EdgeUI.js b/services/static-webserver/client/source/class/osparc/workbench/EdgeUI.js index 01dd4990fab..40e72e134f9 100644 --- a/services/static-webserver/client/source/class/osparc/workbench/EdgeUI.js +++ b/services/static-webserver/client/source/class/osparc/workbench/EdgeUI.js @@ -122,6 +122,15 @@ qx.Class.define("osparc.workbench.EdgeUI", { } }, + setHighlighted: function(highlight) { + if (highlight) { + const strongColor = qx.theme.manager.Color.getInstance().resolve("strong-main"); + osparc.wrapper.Svg.updateCurveColor(this.getRepresentation(), strongColor); + } else { + this.__updateEdgeState(); + } + }, + getEdgeId: function() { return this.getEdge().getEdgeId(); }, diff --git a/services/static-webserver/client/source/class/osparc/workbench/ServiceCatalog.js b/services/static-webserver/client/source/class/osparc/workbench/ServiceCatalog.js index a82a513cd3d..faf60dd0034 100644 --- a/services/static-webserver/client/source/class/osparc/workbench/ServiceCatalog.js +++ b/services/static-webserver/client/source/class/osparc/workbench/ServiceCatalog.js @@ -303,7 +303,9 @@ qx.Class.define("osparc.workbench.ServiceCatalog", { const title = this.tr("Service information"); const width = osparc.info.CardLarge.WIDTH; const height = osparc.info.CardLarge.HEIGHT; - osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height); + osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height).set({ + maxHeight: height, + }); }, __onCancel: function() { diff --git a/services/static-webserver/client/source/class/osparc/workbench/WorkbenchUI.js b/services/static-webserver/client/source/class/osparc/workbench/WorkbenchUI.js index a11cb9ceb10..21c55e487d1 100644 --- a/services/static-webserver/client/source/class/osparc/workbench/WorkbenchUI.js +++ b/services/static-webserver/client/source/class/osparc/workbench/WorkbenchUI.js @@ -484,7 +484,7 @@ qx.Class.define("osparc.workbench.WorkbenchUI", { const allChildren = Array.from(this.getContentElement().getDomElement().getElementsByTagName("*")); const nodesAndSuspicious = allChildren.filter(child => parseInt(child.style.zIndex) >= 100000); nodesAndSuspicious.forEach(child => { - if (child.className !== "qx-window-small-cap") { + if (child.className !== "qx-node-ui-cap") { console.warn("moving undesired element to background"); child.style.zIndex = "1"; } @@ -654,6 +654,25 @@ qx.Class.define("osparc.workbench.WorkbenchUI", { nodeUI.addListener("infoNode", e => this.__openNodeInfo(e.getData()), this); nodeUI.addListener("removeNode", e => this.fireDataEvent("removeNode", e.getData()), this); + if (nodeUI.getNode().getPropsForm()) { + nodeUI.getNode().getPropsForm().addListener("highlightEdge", e => { + const { + highlight, + fromNodeId, + toNodeId, + } = e.getData(); + const edgeFound = this.__edgesUI.find(edgeUI => { + const edge = edgeUI.getEdge(); + const inputNode = edge.getInputNode(); + const outputNode = edge.getOutputNode(); + return (inputNode.getNodeId() === fromNodeId && outputNode.getNodeId() === toNodeId) + }); + if (edgeFound) { + edgeFound.setHighlighted(highlight); + } + }); + } + return nodeUI; }, @@ -1654,7 +1673,9 @@ qx.Class.define("osparc.workbench.WorkbenchUI", { const title = this.tr("Service information"); const width = osparc.info.CardLarge.WIDTH; const height = osparc.info.CardLarge.HEIGHT; - osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height); + osparc.ui.window.Window.popUpInWindow(serviceDetails, title, width, height).set({ + maxHeight: height + }); } }, @@ -2002,7 +2023,7 @@ qx.Class.define("osparc.workbench.WorkbenchUI", { filePicker.addListener("fileUploaded", () => this.fireDataEvent("nodeSelected", nodeUI.getNodeId()), this); } } else { - osparc.FlashMessenger.getInstance().logAs(this.tr("Only one file at a time is accepted."), "ERROR"); + osparc.FlashMessenger.getInstance().logAs(osparc.file.FileDrop.ONE_FILE_ONLY, "ERROR"); } } else { osparc.FlashMessenger.getInstance().logAs(this.tr("Folders are not accepted. You might want to upload a zip file."), "ERROR"); From 9c6068d7d668704eb53ffff420acb589aa0d853b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:46:35 +0100 Subject: [PATCH 13/22] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Enhanced=20groups/or?= =?UTF-8?q?ganizations=20web-api=20specs=20and=20validation=20=20?= =?UTF-8?q?=F0=9F=9A=A8=20(#6640)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/specs/web-server/_groups.py | 101 ++++-- .../api_schemas_webserver/groups.py | 78 ++++- .../models_library/utils/common_validators.py | 35 ++ .../test_api_schemas_webserver_groups.py | 27 ++ .../tests/test_archiving_utils.py | 9 +- .../service-library/tests/test_async_utils.py | 20 +- .../tests/unit/test_api_rest_containers.py | 5 +- services/storage/tests/unit/test_utils.py | 18 +- .../api/v0/openapi.yaml | 304 +++++++++++------- .../simcore_service_webserver/groups/_db.py | 38 ++- .../groups/_handlers.py | 113 ++++--- .../groups/_utils.py | 5 +- .../simcore_service_webserver/groups/api.py | 8 +- .../groups/exceptions.py | 20 +- .../users/schemas.py | 4 +- .../tests/unit/isolated/test_groups_models.py | 6 +- .../tests/unit/with_dbs/01/test_groups.py | 10 +- 17 files changed, 530 insertions(+), 271 deletions(-) create mode 100644 packages/models-library/tests/test_api_schemas_webserver_groups.py diff --git a/api/specs/web-server/_groups.py b/api/specs/web-server/_groups.py index 02c04b663fa..1f8d7e15f56 100644 --- a/api/specs/web-server/_groups.py +++ b/api/specs/web-server/_groups.py @@ -8,14 +8,21 @@ from fastapi import APIRouter, Depends, status from models_library.api_schemas_webserver.groups import ( - AllUsersGroups, + GroupCreate, + GroupGet, + GroupUpdate, GroupUserGet, - UsersGroup, + MyGroupsGet, ) from models_library.generics import Envelope -from models_library.users import GroupID, UserID from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.groups._handlers import _ClassifiersQuery +from simcore_service_webserver.groups._handlers import ( + GroupUserAdd, + GroupUserUpdate, + _ClassifiersQuery, + _GroupPathParams, + _GroupUserPathParams, +) from simcore_service_webserver.scicrunch.models import ResearchResource, ResourceHit router = APIRouter( @@ -28,51 +35,66 @@ @router.get( "/groups", - response_model=Envelope[AllUsersGroups], + response_model=Envelope[MyGroupsGet], ) async def list_groups(): - ... + """ + List all groups (organizations, primary, everyone and products) I belong to + """ @router.post( "/groups", - response_model=Envelope[UsersGroup], + response_model=Envelope[GroupGet], status_code=status.HTTP_201_CREATED, ) -async def create_group(): - ... +async def create_group(_b: GroupCreate): + """ + Creates an organization group + """ @router.get( "/groups/{gid}", - response_model=Envelope[UsersGroup], + response_model=Envelope[GroupGet], ) -async def get_group(gid: GroupID): - ... +async def get_group(_p: Annotated[_GroupPathParams, Depends()]): + """ + Get an organization group + """ @router.patch( "/groups/{gid}", - response_model=Envelope[UsersGroup], + response_model=Envelope[GroupGet], ) -async def update_group(gid: GroupID, _update: UsersGroup): - ... +async def update_group( + _p: Annotated[_GroupPathParams, Depends()], + _b: GroupUpdate, +): + """ + Updates organization groups + """ @router.delete( "/groups/{gid}", status_code=status.HTTP_204_NO_CONTENT, ) -async def delete_group(gid: GroupID): - ... +async def delete_group(_p: Annotated[_GroupPathParams, Depends()]): + """ + Deletes organization groups + """ @router.get( "/groups/{gid}/users", response_model=Envelope[list[GroupUserGet]], ) -async def get_group_users(gid: GroupID): - ... +async def get_all_group_users(_p: Annotated[_GroupPathParams, Depends()]): + """ + Gets users in organization groups + """ @router.post( @@ -80,10 +102,12 @@ async def get_group_users(gid: GroupID): status_code=status.HTTP_204_NO_CONTENT, ) async def add_group_user( - gid: GroupID, - _new: GroupUserGet, + _p: Annotated[_GroupPathParams, Depends()], + _b: GroupUserAdd, ): - ... + """ + Adds a user to an organization group + """ @router.get( @@ -91,10 +115,11 @@ async def add_group_user( response_model=Envelope[GroupUserGet], ) async def get_group_user( - gid: GroupID, - uid: UserID, + _p: Annotated[_GroupUserPathParams, Depends()], ): - ... + """ + Gets specific user in an organization group + """ @router.patch( @@ -102,12 +127,12 @@ async def get_group_user( response_model=Envelope[GroupUserGet], ) async def update_group_user( - gid: GroupID, - uid: UserID, - _update: GroupUserGet, + _p: Annotated[_GroupUserPathParams, Depends()], + _b: GroupUserUpdate, ): - # FIXME: update type - ... + """ + Updates user (access-rights) to an organization group + """ @router.delete( @@ -115,10 +140,16 @@ async def update_group_user( status_code=status.HTTP_204_NO_CONTENT, ) async def delete_group_user( - gid: GroupID, - uid: UserID, + _p: Annotated[_GroupUserPathParams, Depends()], ): - ... + """ + Removes a user from an organization group + """ + + +# +# Classifiers +# @router.get( @@ -126,8 +157,8 @@ async def delete_group_user( response_model=Envelope[dict[str, Any]], ) async def get_group_classifiers( - gid: GroupID, - _query: Annotated[_ClassifiersQuery, Depends()], + _p: Annotated[_GroupPathParams, Depends()], + _q: Annotated[_ClassifiersQuery, Depends()], ): ... diff --git a/packages/models-library/src/models_library/api_schemas_webserver/groups.py b/packages/models-library/src/models_library/api_schemas_webserver/groups.py index e0b6d3fbb37..55107be55c5 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/groups.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/groups.py @@ -1,13 +1,20 @@ from contextlib import suppress from typing import Any, ClassVar -from pydantic import AnyUrl, BaseModel, Field, ValidationError, parse_obj_as, validator +from pydantic import ( + AnyUrl, + BaseModel, + Field, + ValidationError, + parse_obj_as, + root_validator, + validator, +) from ..emails import LowerCaseEmailStr - -# -# GROUPS MODELS defined in OPENAPI specs -# +from ..users import UserID +from ..utils.common_validators import create__check_only_one_is_set__root_validator +from ._base import InputSchema, OutputSchema class GroupAccessRights(BaseModel): @@ -29,7 +36,7 @@ class Config: } -class UsersGroup(BaseModel): +class GroupGet(OutputSchema): gid: int = Field(..., description="the group ID") label: str = Field(..., description="the group name") description: str = Field(..., description="the group description") @@ -45,7 +52,7 @@ class UsersGroup(BaseModel): @validator("thumbnail", pre=True) @classmethod - def sanitize_legacy_data(cls, v): + def _sanitize_legacy_data(cls, v): if v: # Enforces null if thumbnail is not valid URL or empty with suppress(ValidationError): @@ -86,11 +93,23 @@ class Config: } -class AllUsersGroups(BaseModel): - me: UsersGroup | None = None - organizations: list[UsersGroup] | None = None - all: UsersGroup | None = None - product: UsersGroup | None = None +class GroupCreate(InputSchema): + label: str + description: str + thumbnail: AnyUrl | None = None + + +class GroupUpdate(InputSchema): + label: str | None = None + description: str | None = None + thumbnail: AnyUrl | None = None + + +class MyGroupsGet(OutputSchema): + me: GroupGet + organizations: list[GroupGet] | None = None + all: GroupGet + product: GroupGet | None = None class Config: schema_extra: ClassVar[dict[str, Any]] = { @@ -158,3 +177,38 @@ class Config: }, } } + + +class GroupUserAdd(InputSchema): + """ + Identify the user with either `email` or `uid` — only one. + """ + + uid: UserID | None = None + email: LowerCaseEmailStr | None = None + + _check_uid_or_email = root_validator(allow_reuse=True)( + create__check_only_one_is_set__root_validator(["uid", "email"]) + ) + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [{"uid": 42}, {"email": "foo@email.com"}] + } + + +class GroupUserUpdate(InputSchema): + # NOTE: since it is a single item, it is required. Cannot + # update for the moment partial attributes e.g. {read: False} + access_rights: GroupAccessRights + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "example": { + "accessRights": { + "read": True, + "write": False, + "delete": False, + }, + } + } diff --git a/packages/models-library/src/models_library/utils/common_validators.py b/packages/models-library/src/models_library/utils/common_validators.py index c60586dfa92..f1d754de5dc 100644 --- a/packages/models-library/src/models_library/utils/common_validators.py +++ b/packages/models-library/src/models_library/utils/common_validators.py @@ -16,6 +16,8 @@ class MyModel(BaseModel): """ import enum +import functools +import operator from typing import Any @@ -69,3 +71,36 @@ def null_or_none_str_to_none_validator(value: Any): if isinstance(value, str) and value.lower() in ("null", "none"): return None return value + + +def create__check_only_one_is_set__root_validator(alternative_field_names: list[str]): + """Ensure exactly one and only one of the alternatives is set + + NOTE: a field is considered here `unset` when it is `not None`. When None + is used to indicate something else, please do not use this validator. + + This is useful when you want to give the client alternative + ways to set the same thing e.g. set the user by email or id or username + and each of those has a different field + + NOTE: Alternatevely, the previous example can also be solved using a + single field as `user: Email | UserID | UserName` + + SEE test_uid_or_email_are_set.py for more details + """ + + def _validator(cls, values): + assert set(alternative_field_names).issubset(cls.__fields__) # nosec + + got = { + field_name: values.get(field_name) for field_name in alternative_field_names + } + + if not functools.reduce(operator.xor, (v is not None for v in got.values())): + msg = ( + f"Either { 'or'.join(got.keys()) } must be set, but not both. Got {got}" + ) + raise ValueError(msg) + return values + + return _validator diff --git a/packages/models-library/tests/test_api_schemas_webserver_groups.py b/packages/models-library/tests/test_api_schemas_webserver_groups.py new file mode 100644 index 00000000000..c254ef40cf4 --- /dev/null +++ b/packages/models-library/tests/test_api_schemas_webserver_groups.py @@ -0,0 +1,27 @@ +from typing import Any + +import pytest +from models_library.api_schemas_webserver.groups import GroupUserAdd +from pydantic import ValidationError + +unset = object() + + +@pytest.mark.parametrize("uid", [1, None, unset]) +@pytest.mark.parametrize("email", ["user@foo.com", None, unset]) +def test_uid_or_email_are_set(uid: Any, email: Any): + kwargs = {} + if uid != unset: + kwargs["uid"] = uid + if email != unset: + kwargs["email"] = email + + none_are_defined = kwargs.get("uid") is None and kwargs.get("email") is None + both_are_defined = kwargs.get("uid") is not None and kwargs.get("email") is not None + + if none_are_defined or both_are_defined: + with pytest.raises(ValidationError, match="not both"): + GroupUserAdd(**kwargs) + else: + got = GroupUserAdd(**kwargs) + assert bool(got.email) ^ bool(got.uid) diff --git a/packages/service-library/tests/test_archiving_utils.py b/packages/service-library/tests/test_archiving_utils.py index 3996be43ca5..3a94c9fd22c 100644 --- a/packages/service-library/tests/test_archiving_utils.py +++ b/packages/service-library/tests/test_archiving_utils.py @@ -7,7 +7,6 @@ import hashlib import itertools import os -import random import secrets import string import tempfile @@ -35,14 +34,14 @@ def _print_tree(path: Path, level=0): @pytest.fixture -def dir_with_random_content() -> Iterable[Path]: +def dir_with_random_content(faker: Faker) -> Iterable[Path]: def random_string(length: int) -> str: return "".join(secrets.choice(string.ascii_letters) for i in range(length)) def make_files_in_dir(dir_path: Path, file_count: int) -> None: for _ in range(file_count): (dir_path / f"{random_string(8)}.bin").write_bytes( - os.urandom(random.randint(1, 10)) + os.urandom(faker.random_int(1, 10)) ) def ensure_dir(path_to_ensure: Path) -> Path: @@ -53,13 +52,13 @@ def make_subdirectory_with_content(subdir_name: Path, max_file_count: int) -> No subdir_name = ensure_dir(subdir_name) make_files_in_dir( dir_path=subdir_name, - file_count=random.randint(1, max_file_count), + file_count=faker.random_int(1, max_file_count), ) def make_subdirectories_with_content( subdir_name: Path, max_subdirectories_count: int, max_file_count: int ) -> None: - subdirectories_count = random.randint(1, max_subdirectories_count) + subdirectories_count = faker.random_int(1, max_subdirectories_count) for _ in range(subdirectories_count): make_subdirectory_with_content( subdir_name=subdir_name / f"{random_string(4)}", diff --git a/packages/service-library/tests/test_async_utils.py b/packages/service-library/tests/test_async_utils.py index cf2d051fe95..902cbcc9b82 100644 --- a/packages/service-library/tests/test_async_utils.py +++ b/packages/service-library/tests/test_async_utils.py @@ -8,7 +8,7 @@ from collections import deque from dataclasses import dataclass from time import time -from typing import Any, Optional +from typing import Any import pytest from faker import Faker @@ -59,24 +59,25 @@ def _compensate_for_slow_systems(number: float) -> float: async def test_context_aware_dispatch( - sleep_duration: float, - ensure_run_in_sequence_context_is_empty: None, + sleep_duration: float, ensure_run_in_sequence_context_is_empty: None, faker: Faker ) -> None: @run_sequentially_in_context(target_args=["c1", "c2", "c3"]) async def orderly(c1: Any, c2: Any, c3: Any, control: Any) -> None: _ = (c1, c2, c3) await asyncio.sleep(sleep_duration) - context = dict(c1=c1, c2=c2, c3=c3) + context = {"c1": c1, "c2": c2, "c3": c3} await locked_stores[make_key_from_context(context)].push(control) def make_key_from_context(context: dict) -> str: return ".".join([f"{k}:{v}" for k, v in context.items()]) def make_context(): - return dict( - c1=random.randint(0, 10), c2=random.randint(0, 10), c3=random.randint(0, 10) - ) + return { + "c1": faker.random_int(0, 10), + "c2": faker.random_int(0, 10), + "c3": faker.random_int(0, 10), + } contexts = [make_context() for _ in range(10)] @@ -116,7 +117,8 @@ class DidFailException(Exception): @run_sequentially_in_context(target_args=["will_fail"]) async def sometimes_failing(will_fail: bool) -> bool: if will_fail: - raise DidFailException("I was instructed to fail") + msg = "I was instructed to fail" + raise DidFailException(msg) return True for x in range(100): @@ -201,7 +203,7 @@ class ObjectWithPropos: @run_sequentially_in_context(target_args=["object_with_props.attr1"]) async def test_attribute( - object_with_props: ObjectWithPropos, other_attr: Optional[int] = None + object_with_props: ObjectWithPropos, other_attr: int | None = None ) -> str: return object_with_props.attr1 diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py index f16b883de15..83e7a803f10 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py @@ -5,7 +5,6 @@ import asyncio import json -import random from collections.abc import AsyncIterable from inspect import signature from pathlib import Path @@ -303,9 +302,9 @@ async def attachable_networks_and_ids(faker: Faker) -> AsyncIterable[dict[str, s @pytest.fixture -def mock_aiodocker_containers_get(mocker: MockerFixture) -> int: +def mock_aiodocker_containers_get(mocker: MockerFixture, faker: Faker) -> int: """raises a DockerError with a random HTTP status which is also returned""" - mock_status_code = random.randint(1, 999) # noqa: S311 + mock_status_code = faker.random_int(1, 999) async def mock_get(*args: str, **kwargs: Any) -> None: raise aiodocker.exceptions.DockerError( diff --git a/services/storage/tests/unit/test_utils.py b/services/storage/tests/unit/test_utils.py index 1b71a1c29d5..03dc7e2e630 100644 --- a/services/storage/tests/unit/test_utils.py +++ b/services/storage/tests/unit/test_utils.py @@ -6,9 +6,8 @@ import datetime -import random +from collections.abc import Callable from pathlib import Path -from typing import Callable from uuid import uuid4 import pytest @@ -17,6 +16,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from pydantic import ByteSize, HttpUrl, parse_obj_as +from pytest_simcore.helpers.faker_factories import DEFAULT_FAKER from simcore_service_storage.constants import S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID from simcore_service_storage.models import ETag, FileMetaData, S3BucketName, UploadID from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager @@ -46,24 +46,30 @@ async def test_download_files(tmp_path: Path, httpbin_base_url: HttpUrl): [ (-1, None, None, None, False), (0, None, None, None, False), - (random.randint(1, 1000000), None, None, None, False), + (DEFAULT_FAKER.random_int(1, 1000000), None, None, None, False), (-1, "some_valid_entity_tag", None, None, False), (0, "some_valid_entity_tag", None, None, False), ( - random.randint(1, 1000000), + DEFAULT_FAKER.random_int(1, 1000000), "some_valid_entity_tag", "som_upload_id", None, False, ), ( - random.randint(1, 1000000), + DEFAULT_FAKER.random_int(1, 1000000), "some_valid_entity_tag", None, datetime.datetime.utcnow(), False, ), - (random.randint(1, 1000000), "some_valid_entity_tag", None, None, True), + ( + DEFAULT_FAKER.random_int(1, 1000000), + "some_valid_entity_tag", + None, + None, + True, + ), ], ) def test_file_entry_valid( diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 0df5e076a76..82a18aebb3b 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -454,6 +454,8 @@ paths: tags: - groups summary: List Groups + description: List all groups (organizations, primary, everyone and products) + I belong to operationId: list_groups responses: '200': @@ -461,24 +463,32 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Envelope_AllUsersGroups_' + $ref: '#/components/schemas/Envelope_MyGroupsGet_' post: tags: - groups summary: Create Group + description: Creates an organization group operationId: create_group + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GroupCreate' + required: true responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/Envelope_UsersGroup_' + $ref: '#/components/schemas/Envelope_GroupGet_' /v0/groups/{gid}: get: tags: - groups summary: Get Group + description: Get an organization group operationId: get_group parameters: - required: true @@ -495,11 +505,12 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Envelope_UsersGroup_' + $ref: '#/components/schemas/Envelope_GroupGet_' delete: tags: - groups summary: Delete Group + description: Deletes organization groups operationId: delete_group parameters: - required: true @@ -517,6 +528,7 @@ paths: tags: - groups summary: Update Group + description: Updates organization groups operationId: update_group parameters: - required: true @@ -531,7 +543,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UsersGroup' + $ref: '#/components/schemas/GroupUpdate' required: true responses: '200': @@ -539,13 +551,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Envelope_UsersGroup_' + $ref: '#/components/schemas/Envelope_GroupGet_' /v0/groups/{gid}/users: get: tags: - groups - summary: Get Group Users - operationId: get_group_users + summary: Get All Group Users + description: Gets users in organization groups + operationId: get_all_group_users parameters: - required: true schema: @@ -566,6 +579,7 @@ paths: tags: - groups summary: Add Group User + description: Adds a user to an organization group operationId: add_group_user parameters: - required: true @@ -580,7 +594,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/GroupUserGet' + $ref: '#/components/schemas/GroupUserAdd' required: true responses: '204': @@ -590,6 +604,7 @@ paths: tags: - groups summary: Get Group User + description: Gets specific user in an organization group operationId: get_group_user parameters: - required: true @@ -619,6 +634,7 @@ paths: tags: - groups summary: Delete Group User + description: Removes a user from an organization group operationId: delete_group_user parameters: - required: true @@ -644,6 +660,7 @@ paths: tags: - groups summary: Update Group User + description: Updates user (access-rights) to an organization group operationId: update_group_user parameters: - required: true @@ -666,7 +683,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/GroupUserGet' + $ref: '#/components/schemas/GroupUserUpdate' required: true responses: '200': @@ -4311,7 +4328,7 @@ paths: '403': description: ProjectInvalidRightsError '404': - description: UserDefaultWalletNotFoundError, ProjectNotFoundError + description: ProjectNotFoundError, UserDefaultWalletNotFoundError '409': description: ProjectTooManyProjectOpenedError '422': @@ -6022,54 +6039,6 @@ components: queued: title: Queued type: boolean - AllUsersGroups: - title: AllUsersGroups - type: object - properties: - me: - $ref: '#/components/schemas/UsersGroup' - organizations: - title: Organizations - type: array - items: - $ref: '#/components/schemas/UsersGroup' - all: - $ref: '#/components/schemas/UsersGroup' - product: - $ref: '#/components/schemas/UsersGroup' - example: - me: - gid: '27' - label: A user - description: A very special user - accessRights: - read: true - write: true - delete: true - organizations: - - gid: '15' - label: ITIS Foundation - description: The Foundation for Research on Information Technologies in - Society - accessRights: - read: true - write: false - delete: false - - gid: '16' - label: Blue Fundation - description: Some foundation - accessRights: - read: true - write: false - delete: false - all: - gid: '0' - label: All - description: Open to all users - accessRights: - read: true - write: false - delete: false Annotation: title: Annotation required: @@ -7134,14 +7103,6 @@ components: type: object properties: {} additionalProperties: false - Envelope_AllUsersGroups_: - title: Envelope[AllUsersGroups] - type: object - properties: - data: - $ref: '#/components/schemas/AllUsersGroups' - error: - title: Error Envelope_AnyUrl_: title: Envelope[AnyUrl] type: object @@ -7290,6 +7251,14 @@ components: $ref: '#/components/schemas/GetWalletAutoRecharge' error: title: Error + Envelope_GroupGet_: + title: Envelope[GroupGet] + type: object + properties: + data: + $ref: '#/components/schemas/GroupGet' + error: + title: Error Envelope_GroupUserGet_: title: Envelope[GroupUserGet] type: object @@ -7338,6 +7307,14 @@ components: $ref: '#/components/schemas/LoginNextPage' error: title: Error + Envelope_MyGroupsGet_: + title: Envelope[MyGroupsGet] + type: object + properties: + data: + $ref: '#/components/schemas/MyGroupsGet' + error: + title: Error Envelope_NodeCreated_: title: Envelope[NodeCreated] type: object @@ -7587,14 +7564,6 @@ components: $ref: '#/components/schemas/UserProfile' error: title: Error - Envelope_UsersGroup_: - title: Envelope[UsersGroup] - type: object - properties: - data: - $ref: '#/components/schemas/UsersGroup' - error: - title: Error Envelope_WalletGetWithAvailableCredits_: title: Envelope[WalletGetWithAvailableCredits] type: object @@ -8657,6 +8626,91 @@ components: title: Delete type: boolean description: defines acesss rights for the user + GroupCreate: + title: GroupCreate + required: + - label + - description + type: object + properties: + label: + title: Label + type: string + description: + title: Description + type: string + thumbnail: + title: Thumbnail + maxLength: 65536 + minLength: 1 + type: string + format: uri + GroupGet: + title: GroupGet + required: + - gid + - label + - description + - accessRights + type: object + properties: + gid: + title: Gid + type: integer + description: the group ID + label: + title: Label + type: string + description: the group name + description: + title: Description + type: string + description: the group description + thumbnail: + title: Thumbnail + maxLength: 65536 + minLength: 1 + type: string + description: url to the group thumbnail + format: uri + accessRights: + $ref: '#/components/schemas/GroupAccessRights' + inclusionRules: + title: Inclusionrules + type: object + additionalProperties: + type: string + description: Maps user's column and regular expression + GroupUpdate: + title: GroupUpdate + type: object + properties: + label: + title: Label + type: string + description: + title: Description + type: string + thumbnail: + title: Thumbnail + maxLength: 65536 + minLength: 1 + type: string + format: uri + GroupUserAdd: + title: GroupUserAdd + type: object + properties: + uid: + title: Uid + exclusiveMinimum: true + type: integer + minimum: 0 + email: + title: Email + type: string + format: email + description: "Identify the user with either `email` or `uid` \u2014 only one." GroupUserGet: title: GroupUserGet required: @@ -8701,6 +8755,19 @@ components: read: true write: false delete: false + GroupUserUpdate: + title: GroupUserUpdate + required: + - accessRights + type: object + properties: + accessRights: + $ref: '#/components/schemas/GroupAccessRights' + example: + accessRights: + read: true + write: false + delete: false HardwareInfo: title: HardwareInfo required: @@ -8993,6 +9060,57 @@ components: type: string format: color additionalProperties: false + MyGroupsGet: + title: MyGroupsGet + required: + - me + - all + type: object + properties: + me: + $ref: '#/components/schemas/GroupGet' + organizations: + title: Organizations + type: array + items: + $ref: '#/components/schemas/GroupGet' + all: + $ref: '#/components/schemas/GroupGet' + product: + $ref: '#/components/schemas/GroupGet' + example: + me: + gid: '27' + label: A user + description: A very special user + accessRights: + read: true + write: true + delete: true + organizations: + - gid: '15' + label: ITIS Foundation + description: The Foundation for Research on Information Technologies in + Society + accessRights: + read: true + write: false + delete: false + - gid: '16' + label: Blue Fundation + description: Some foundation + accessRights: + read: true + write: false + delete: false + all: + gid: '0' + label: All + description: Open to all users + accessRights: + read: true + write: false + delete: false NoAuthentication: title: NoAuthentication type: object @@ -10156,7 +10274,7 @@ components: - ADMIN type: string groups: - $ref: '#/components/schemas/AllUsersGroups' + $ref: '#/components/schemas/MyGroupsGet' gravatar_id: title: Gravatar Id type: string @@ -12587,42 +12705,6 @@ components: - DELETED type: string description: An enumeration. - UsersGroup: - title: UsersGroup - required: - - gid - - label - - description - - accessRights - type: object - properties: - gid: - title: Gid - type: integer - description: the group ID - label: - title: Label - type: string - description: the group name - description: - title: Description - type: string - description: the group description - thumbnail: - title: Thumbnail - maxLength: 65536 - minLength: 1 - type: string - description: url to the group thumbnail - format: uri - accessRights: - $ref: '#/components/schemas/GroupAccessRights' - inclusionRules: - title: Inclusionrules - type: object - additionalProperties: - type: string - description: Maps user's column and regular expression Viewer: title: Viewer required: diff --git a/services/web/server/src/simcore_service_webserver/groups/_db.py b/services/web/server/src/simcore_service_webserver/groups/_db.py index 38bbb4e7d7c..d27d7a8a441 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_db.py +++ b/services/web/server/src/simcore_service_webserver/groups/_db.py @@ -7,6 +7,7 @@ from models_library.groups import GroupAtDB from models_library.users import GroupID, UserID from pydantic import parse_obj_as +from simcore_postgres_database.errors import UniqueViolation from simcore_postgres_database.utils_products import get_or_create_product_group from sqlalchemy import and_, literal_column from sqlalchemy.dialects.postgresql import insert @@ -20,7 +21,11 @@ convert_groups_db_to_schema, convert_groups_schema_to_db, ) -from .exceptions import GroupNotFoundError, UserInGroupNotFoundError +from .exceptions import ( + GroupNotFoundError, + UserAlreadyInGroupError, + UserInGroupNotFoundError, +) _DEFAULT_PRODUCT_GROUP_ACCESS_RIGHTS = AccessRightsDict( read=False, @@ -50,7 +55,7 @@ async def _get_user_group( ) group = await result.fetchone() if not group: - raise GroupNotFoundError(gid) + raise GroupNotFoundError(gid=gid) assert isinstance(group, RowProxy) # nosec return group @@ -305,24 +310,31 @@ async def add_new_user_in_group( # first check if the group exists group: RowProxy = await _get_user_group(conn, user_id, gid) check_group_permissions(group, user_id, gid, "write") + # now check the new user exists users_count = await conn.scalar( sa.select(sa.func.count()).where(users.c.id == new_user_id) ) if not users_count: assert new_user_id is not None # nosec - raise UserInGroupNotFoundError(new_user_id, gid) + raise UserInGroupNotFoundError(uid=new_user_id, gid=gid) # add the new user to the group now user_access_rights = _DEFAULT_GROUP_READ_ACCESS_RIGHTS if access_rights: user_access_rights.update(access_rights) - await conn.execute( - # pylint: disable=no-value-for-parameter - user_to_groups.insert().values( - uid=new_user_id, gid=group.gid, access_rights=user_access_rights + + try: + await conn.execute( + # pylint: disable=no-value-for-parameter + user_to_groups.insert().values( + uid=new_user_id, gid=group.gid, access_rights=user_access_rights + ) ) - ) + except UniqueViolation as exc: + raise UserAlreadyInGroupError( + uid=new_user_id, gid=gid, user_id=user_id, access_rights=access_rights + ) from exc async def _get_user_in_group_permissions( @@ -336,7 +348,7 @@ async def _get_user_in_group_permissions( ) the_user: RowProxy | None = await result.fetchone() if not the_user: - raise UserInGroupNotFoundError(the_user_id_in_group, gid) + raise UserInGroupNotFoundError(uid=the_user_id_in_group, gid=gid) return the_user @@ -358,8 +370,12 @@ async def update_user_in_group( user_id: UserID, gid: GroupID, the_user_id_in_group: int, - new_values_for_user_in_group: dict, + access_rights: dict, ) -> dict[str, str]: + if not access_rights: + msg = f"Cannot update empty {access_rights}" + raise ValueError(msg) + # first check if the group exists group: RowProxy = await _get_user_group(conn, user_id, gid) check_group_permissions(group, user_id, gid, "write") @@ -368,7 +384,7 @@ async def update_user_in_group( conn, gid, the_user_id_in_group ) # modify the user access rights - new_db_values = {"access_rights": new_values_for_user_in_group["accessRights"]} + new_db_values = {"access_rights": access_rights} await conn.execute( # pylint: disable=no-value-for-parameter user_to_groups.update() diff --git a/services/web/server/src/simcore_service_webserver/groups/_handlers.py b/services/web/server/src/simcore_service_webserver/groups/_handlers.py index b6284bed8bf..32a9c4c1a40 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/groups/_handlers.py @@ -5,21 +5,23 @@ from aiohttp import web from models_library.api_schemas_webserver.groups import ( - AllUsersGroups, + GroupCreate, + GroupGet, + GroupUpdate, + GroupUserAdd, GroupUserGet, - UsersGroup, + GroupUserUpdate, + MyGroupsGet, ) -from models_library.emails import LowerCaseEmailStr from models_library.users import GroupID, UserID -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, Extra, Field, parse_obj_as from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( + parse_request_body_as, parse_request_path_parameters_as, parse_request_query_parameters_as, ) from servicelib.aiohttp.typing_extension import Handler -from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from .._constants import RQ_PRODUCT_KEY, RQT_USERID_KEY from .._meta import API_VTAG @@ -36,6 +38,7 @@ from ._classifiers import GroupClassifierRepository, build_rrids_tree_view from .exceptions import ( GroupNotFoundError, + UserAlreadyInGroupError, UserInGroupNotFoundError, UserInsufficientRightsError, ) @@ -55,13 +58,21 @@ async def wrapper(request: web.Request) -> web.StreamResponse: return await handler(request) except UserNotFoundError as exc: - raise web.HTTPNotFound(reason=f"User {exc.uid} not found") from exc + raise web.HTTPNotFound( + reason=f"User {exc.uid or exc.email} not found" + ) from exc except GroupNotFoundError as exc: - raise web.HTTPNotFound(reason=f"Group {exc.gid} not found") from exc + gid = getattr(exc, "gid", "") + raise web.HTTPNotFound(reason=f"Group {gid} not found") from exc except UserInGroupNotFoundError as exc: - raise web.HTTPNotFound(reason=f"User not found in group {exc.gid}") from exc + gid = getattr(exc, "gid", "") + raise web.HTTPNotFound(reason=f"User not found in group {gid}") from exc + + except UserAlreadyInGroupError as exc: + gid = getattr(exc, "gid", "") + raise web.HTTPConflict(reason=f"User is already in group {gid}") from exc except UserInsufficientRightsError as exc: raise web.HTTPForbidden from exc @@ -77,11 +88,9 @@ async def wrapper(request: web.Request) -> web.StreamResponse: @permission_required("groups.read") @_handle_groups_exceptions async def list_groups(request: web.Request): - """Lists my groups - - List of the groups I belonged to """ - + List all groups (organizations, primary, everyone and products) I belong to + """ product: Product = get_current_product(request) req_ctx = _GroupsRequestContext.parse_obj(request) @@ -89,7 +98,7 @@ async def list_groups(request: web.Request): request.app, req_ctx.user_id ) - result = { + my_group = { "me": primary_group, "organizations": user_groups, "all": all_group, @@ -98,14 +107,20 @@ async def list_groups(request: web.Request): if product.group_id: with suppress(GroupNotFoundError): - result["product"] = await api.get_product_group_for_user( + # Product is optional + my_group["product"] = await api.get_product_group_for_user( app=request.app, user_id=req_ctx.user_id, product_gid=product.group_id, ) - assert parse_obj_as(AllUsersGroups, result) is not None # nosec - return result + assert parse_obj_as(MyGroupsGet, my_group) is not None # nosec + return envelope_json_response(my_group) + + +# +# Organization groups +# class _GroupPathParams(BaseModel): @@ -120,13 +135,13 @@ class Config: @permission_required("groups.read") @_handle_groups_exceptions async def get_group(request: web.Request): - """Get one group details""" + """Get an organization group""" req_ctx = _GroupsRequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) group = await api.get_user_group(request.app, req_ctx.user_id, path_params.gid) - assert parse_obj_as(UsersGroup, group) is not None # nosec - return group + assert parse_obj_as(GroupGet, group) is not None # nosec + return envelope_json_response(group) @routes.post(f"/{API_VTAG}/groups", name="create_group") @@ -134,15 +149,15 @@ async def get_group(request: web.Request): @permission_required("groups.*") @_handle_groups_exceptions async def create_group(request: web.Request): - """Creates organization groups""" + """Creates an organization group""" req_ctx = _GroupsRequestContext.parse_obj(request) - new_group = await request.json() + create = await parse_request_body_as(GroupCreate, request) + new_group = create.dict(exclude_unset=True) created_group = await api.create_user_group(request.app, req_ctx.user_id, new_group) - assert parse_obj_as(UsersGroup, created_group) is not None # nosec - raise web.HTTPCreated( - text=json_dumps({"data": created_group}), content_type=MIMETYPE_APPLICATION_JSON - ) + assert parse_obj_as(GroupGet, created_group) is not None # nosec + + return envelope_json_response(created_group, status_cls=web.HTTPCreated) @routes.patch(f"/{API_VTAG}/groups/{{gid}}", name="update_group") @@ -150,14 +165,16 @@ async def create_group(request: web.Request): @permission_required("groups.*") @_handle_groups_exceptions async def update_group(request: web.Request): + """Updates organization groups""" req_ctx = _GroupsRequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) - new_group_values = await request.json() + update: GroupUpdate = await parse_request_body_as(GroupUpdate, request) + new_group_values = update.dict(exclude_unset=True) updated_group = await api.update_user_group( request.app, req_ctx.user_id, path_params.gid, new_group_values ) - assert parse_obj_as(UsersGroup, updated_group) is not None # nosec + assert parse_obj_as(GroupGet, updated_group) is not None # nosec return envelope_json_response(updated_group) @@ -166,6 +183,7 @@ async def update_group(request: web.Request): @permission_required("groups.*") @_handle_groups_exceptions async def delete_group(request: web.Request): + """Deletes organization groups""" req_ctx = _GroupsRequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) @@ -173,11 +191,17 @@ async def delete_group(request: web.Request): return web.json_response(status=status.HTTP_204_NO_CONTENT) -@routes.get(f"/{API_VTAG}/groups/{{gid}}/users", name="get_group_users") +# +# Users in organization groups (i.e. members of an organization) +# + + +@routes.get(f"/{API_VTAG}/groups/{{gid}}/users", name="get_all_group_users") @login_required @permission_required("groups.*") @_handle_groups_exceptions -async def get_group_users(request: web.Request): +async def get_all_group_users(request: web.Request): + """Gets users in organization groups""" req_ctx = _GroupsRequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) @@ -198,23 +222,14 @@ async def add_group_user(request: web.Request): """ req_ctx = _GroupsRequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) - new_user_in_group = await request.json() - - assert "uid" in new_user_in_group or "email" in new_user_in_group # nosec - - new_user_id = new_user_in_group["uid"] if "uid" in new_user_in_group else None - new_user_email = ( - parse_obj_as(LowerCaseEmailStr, new_user_in_group["email"]) - if "email" in new_user_in_group - else None - ) + added: GroupUserAdd = await parse_request_body_as(GroupUserAdd, request) await api.add_user_in_group( request.app, req_ctx.user_id, path_params.gid, - new_user_id=new_user_id, - new_user_email=new_user_email, + new_user_id=added.uid, + new_user_email=added.email, ) return web.json_response(status=status.HTTP_204_NO_CONTENT) @@ -233,7 +248,7 @@ class Config: @_handle_groups_exceptions async def get_group_user(request: web.Request): """ - Gets specific user in group + Gets specific user in an organization group """ req_ctx = _GroupsRequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(_GroupUserPathParams, request) @@ -249,18 +264,16 @@ async def get_group_user(request: web.Request): @permission_required("groups.*") @_handle_groups_exceptions async def update_group_user(request: web.Request): - """ - Modify specific user in group - """ req_ctx = _GroupsRequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(_GroupUserPathParams, request) - new_values_for_user_in_group = await request.json() + update: GroupUserUpdate = await parse_request_body_as(GroupUserUpdate, request) + user = await api.update_user_in_group( request.app, - req_ctx.user_id, - path_params.gid, - path_params.uid, - new_values_for_user_in_group, + user_id=req_ctx.user_id, + gid=path_params.gid, + the_user_id_in_group=path_params.uid, + access_rights=update.access_rights.dict(), ) assert parse_obj_as(GroupUserGet, user) is not None # nosec return envelope_json_response(user) diff --git a/services/web/server/src/simcore_service_webserver/groups/_utils.py b/services/web/server/src/simcore_service_webserver/groups/_utils.py index 163a4923437..4f0f3ad759f 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_utils.py +++ b/services/web/server/src/simcore_service_webserver/groups/_utils.py @@ -24,8 +24,9 @@ def check_group_permissions( group: RowProxy, user_id: int, gid: int, permission: str ) -> None: if not group.access_rights[permission]: - msg = f"User {user_id} has insufficient rights for {permission} access to group {gid}" - raise UserInsufficientRightsError(msg) + raise UserInsufficientRightsError( + user_id=user_id, gid=gid, permission=permission + ) def convert_groups_db_to_schema( diff --git a/services/web/server/src/simcore_service_webserver/groups/api.py b/services/web/server/src/simcore_service_webserver/groups/api.py index e0a837f8399..f2cbeb00094 100644 --- a/services/web/server/src/simcore_service_webserver/groups/api.py +++ b/services/web/server/src/simcore_service_webserver/groups/api.py @@ -139,7 +139,7 @@ async def add_user_in_group( if not new_user_id and not new_user_email: msg = "Invalid method call, missing user id or user email" - raise GroupsError(msg) + raise GroupsError(msg=msg) async with get_database_engine(app).acquire() as conn: if new_user_email: @@ -148,7 +148,7 @@ async def add_user_in_group( if not new_user_id: msg = "Missing new user in arguments" - raise GroupsError(msg) + raise GroupsError(msg=msg) return await _db.add_new_user_in_group( conn, @@ -173,7 +173,7 @@ async def update_user_in_group( user_id: UserID, gid: GroupID, the_user_id_in_group: int, - new_values_for_user_in_group: dict, + access_rights: dict, ) -> dict[str, str]: async with get_database_engine(app).acquire() as conn: return await _db.update_user_in_group( @@ -181,7 +181,7 @@ async def update_user_in_group( user_id=user_id, gid=gid, the_user_id_in_group=the_user_id_in_group, - new_values_for_user_in_group=new_values_for_user_in_group, + access_rights=access_rights, ) diff --git a/services/web/server/src/simcore_service_webserver/groups/exceptions.py b/services/web/server/src/simcore_service_webserver/groups/exceptions.py index 7556a8cae85..f30bf56de30 100644 --- a/services/web/server/src/simcore_service_webserver/groups/exceptions.py +++ b/services/web/server/src/simcore_service_webserver/groups/exceptions.py @@ -4,28 +4,22 @@ class GroupsError(WebServerBaseError): - msg_template = "{msg}" - - def __init__(self, msg: str | None = None): - super().__init__(msg=msg or "Unexpected error occured in projects subpackage") + msg_template = "Groups plugin errored: {msg}" class GroupNotFoundError(GroupsError): msg_template = "Group with id {gid} not found" - def __init__(self, gid, **extras): - super().__init__(**extras) - self.gid = gid - class UserInsufficientRightsError(GroupsError): - ... + msg = ( + "User {user_id} has insufficient rights for {permission} access to group {gid}" + ) class UserInGroupNotFoundError(GroupsError): msg_template = "User id {uid} in Group {gid} not found" - def __init__(self, uid, gid, **extras): - super().__init__(**extras) - self.uid = uid - self.gid = gid + +class UserAlreadyInGroupError(GroupsError): + msg_template = "User `{uid}` is already in Group `{gid}`" diff --git a/services/web/server/src/simcore_service_webserver/users/schemas.py b/services/web/server/src/simcore_service_webserver/users/schemas.py index 53c9ee9b756..ef8973b3abf 100644 --- a/services/web/server/src/simcore_service_webserver/users/schemas.py +++ b/services/web/server/src/simcore_service_webserver/users/schemas.py @@ -3,7 +3,7 @@ from uuid import UUID from models_library.api_schemas_webserver._base import OutputSchema -from models_library.api_schemas_webserver.groups import AllUsersGroups +from models_library.api_schemas_webserver.groups import MyGroupsGet from models_library.api_schemas_webserver.users_preferences import AggregatedPreferences from models_library.emails import LowerCaseEmailStr from models_library.users import FirstNameStr, LastNameStr, UserID @@ -65,7 +65,7 @@ class ProfileGet(BaseModel): last_name: LastNameStr | None = None login: LowerCaseEmailStr role: Literal["ANONYMOUS", "GUEST", "USER", "TESTER", "PRODUCT_OWNER", "ADMIN"] - groups: AllUsersGroups | None = None + groups: MyGroupsGet | None = None gravatar_id: str | None = None expiration_date: date | None = Field( default=None, diff --git a/services/web/server/tests/unit/isolated/test_groups_models.py b/services/web/server/tests/unit/isolated/test_groups_models.py index 10b49979b1c..3a5fd5e91ce 100644 --- a/services/web/server/tests/unit/isolated/test_groups_models.py +++ b/services/web/server/tests/unit/isolated/test_groups_models.py @@ -1,6 +1,6 @@ import models_library.groups import simcore_postgres_database.models.groups -from models_library.api_schemas_webserver.groups import UsersGroup +from models_library.api_schemas_webserver.groups import GroupGet from models_library.utils.enums import enum_to_dict @@ -14,7 +14,7 @@ def test_models_library_and_postgress_database_enums_are_equivalent(): def test_sanitize_legacy_data(): - users_group_1 = UsersGroup.parse_obj( + users_group_1 = GroupGet.parse_obj( { "gid": "27", "label": "A user", @@ -26,7 +26,7 @@ def test_sanitize_legacy_data(): assert users_group_1.thumbnail is None - users_group_2 = UsersGroup.parse_obj( + users_group_2 = GroupGet.parse_obj( { "gid": "27", "label": "A user", diff --git a/services/web/server/tests/unit/with_dbs/01/test_groups.py b/services/web/server/tests/unit/with_dbs/01/test_groups.py index f6e41225ff7..f616dee6110 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_groups.py +++ b/services/web/server/tests/unit/with_dbs/01/test_groups.py @@ -4,7 +4,6 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -import random from collections.abc import AsyncIterator, Callable from contextlib import AsyncExitStack from copy import deepcopy @@ -160,7 +159,7 @@ async def test_list_groups( ) response = await client.patch( f"{url}", - json={"access_rights": {"read": True, "write": True, "delete": True}}, + json={"accessRights": {"read": True, "write": True, "delete": True}}, ) await assert_status(response, status.HTTP_403_FORBIDDEN) @@ -285,6 +284,7 @@ async def test_add_remove_users_from_group( logged_user: UserInfoDict, user_role: UserRole, expected: ExpectedResponse, + faker: Faker, ): assert client.app new_group = { @@ -295,7 +295,7 @@ async def test_add_remove_users_from_group( } # check that our group does not exist - url = client.app.router["get_group_users"].url_for(gid=new_group["gid"]) + url = client.app.router["get_all_group_users"].url_for(gid=new_group["gid"]) assert f"{url}" == f"/{API_VTAG}/groups/{new_group['gid']}/users" resp = await client.get(f"{url}") data, error = await assert_status(resp, expected.not_found) @@ -323,7 +323,7 @@ async def test_add_remove_users_from_group( assert assigned_group["accessRights"] == _DEFAULT_GROUP_OWNER_ACCESS_RIGHTS # check that our user is in the group of users - get_group_users_url = client.app.router["get_group_users"].url_for( + get_group_users_url = client.app.router["get_all_group_users"].url_for( gid=f"{assigned_group['gid']}" ) assert ( @@ -345,7 +345,7 @@ async def test_add_remove_users_from_group( assert ( f"{add_group_user_url}" == f"/{API_VTAG}/groups/{assigned_group['gid']}/users" ) - num_new_users = random.randint(1, 10) + num_new_users = faker.random_int(1, 10) created_users_list = [] async with AsyncExitStack() as users_stack: From db3aacf3341acde346bf3b6e93c4f164f9480a93 Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:18:19 +0100 Subject: [PATCH 14/22] =?UTF-8?q?=F0=9F=8E=A8=20[Frontend]=20Improve=20Not?= =?UTF-8?q?ification=20texts=20and=20Bell's=20UX=20(#6661)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../class/osparc/dashboard/StudyBrowser.js | 6 + .../class/osparc/desktop/MainPageDesktop.js | 48 +++--- .../source/class/osparc/info/CommentUI.js | 14 +- .../class/osparc/navigation/NavigationBar.js | 12 +- .../class/osparc/notification/Notification.js | 16 ++ .../osparc/notification/NotificationUI.js | 141 ++++++++++++------ .../osparc/notification/Notifications.js | 37 ++++- .../notification/NotificationsButton.js | 7 +- .../client/source/class/osparc/store/Store.js | 21 ++- .../api/v0/openapi.yaml | 32 ++++ .../users/_notifications.py | 6 + .../unit/isolated/test_user_notifications.py | 4 + .../with_dbs/03/test_users__notifications.py | 4 + 13 files changed, 246 insertions(+), 102 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js index 804986735a7..ab29d814808 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js @@ -115,6 +115,12 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { } // "Starting..." page this._hideLoadingPage(); + + // since all the resources (templates, users, orgs...) were already loaded, notifications can be built + osparc.data.Resources.get("notifications") + .then(notifications => { + osparc.notification.Notifications.getInstance().addNotifications(notifications); + }); }) .catch(err => { console.error(err); diff --git a/services/static-webserver/client/source/class/osparc/desktop/MainPageDesktop.js b/services/static-webserver/client/source/class/osparc/desktop/MainPageDesktop.js index e39b9c72357..93f5f50c74d 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/MainPageDesktop.js +++ b/services/static-webserver/client/source/class/osparc/desktop/MainPageDesktop.js @@ -26,31 +26,29 @@ qx.Class.define("osparc.desktop.MainPageDesktop", { this._add(osparc.notification.RibbonNotifications.getInstance()); const navBar = new osparc.navigation.NavigationBar(); - navBar.populateLayout() - .then(() => { - // exclude some items from the navigation bar - navBar.getChildControl("dashboard-label").exclude(); - navBar.getChildControl("dashboard-button").exclude(); - navBar.getChildControl("notifications-button").exclude(); - navBar.getChildControl("help").exclude(); - - // exclude all the menu entries except "log-out" from user menu - const userMenuButton = navBar.getChildControl("user-menu"); - const userMenu = userMenuButton.getMenu(); - // eslint-disable-next-line no-underscore-dangle - const userMenuEntries = userMenu._getCreatedChildControls(); - Object.entries(userMenuEntries).forEach(([id, userMenuEntry]) => { - if (!["mini-profile-view", "po-center", "log-out"].includes(id)) { - userMenuEntry.exclude(); - } - }); - // exclude also the separators - userMenu.getChildren().forEach(child => { - if (child.classname === "qx.ui.menu.Separator") { - child.exclude(); - } - }); - }); + navBar.populateLayout(); + // exclude some items from the navigation bar + navBar.getChildControl("dashboard-label").exclude(); + navBar.getChildControl("dashboard-button").exclude(); + navBar.getChildControl("notifications-button").exclude(); + navBar.getChildControl("help").exclude(); + + // exclude all the menu entries except "log-out" from user menu + const userMenuButton = navBar.getChildControl("user-menu"); + const userMenu = userMenuButton.getMenu(); + // eslint-disable-next-line no-underscore-dangle + const userMenuEntries = userMenu._getCreatedChildControls(); + Object.entries(userMenuEntries).forEach(([id, userMenuEntry]) => { + if (!["mini-profile-view", "po-center", "log-out"].includes(id)) { + userMenuEntry.exclude(); + } + }); + // exclude also the separators + userMenu.getChildren().forEach(child => { + if (child.classname === "qx.ui.menu.Separator") { + child.exclude(); + } + }); this._add(navBar); osparc.MaintenanceTracker.getInstance().startTracker(); diff --git a/services/static-webserver/client/source/class/osparc/info/CommentUI.js b/services/static-webserver/client/source/class/osparc/info/CommentUI.js index 47a005cdb12..62871860da6 100644 --- a/services/static-webserver/client/source/class/osparc/info/CommentUI.js +++ b/services/static-webserver/client/source/class/osparc/info/CommentUI.js @@ -117,14 +117,12 @@ qx.Class.define("osparc.info.CommentUI", { const commentContent = this.getChildControl("comment-content"); commentContent.setValue(this.__comment["contents"]); - osparc.store.Store.getInstance().getUser(this.__comment["user_id"]) - .then(user => { - if (user) { - const userSource = osparc.utils.Avatar.getUrl(user["login"], 32); - thumbnail.setSource(userSource); - userName.setValue(user["label"]); - } - }); + const user = osparc.store.Store.getInstance().getUser(this.__comment["user_id"]) + if (user) { + const userSource = osparc.utils.Avatar.getUrl(user["login"], 32); + thumbnail.setSource(userSource); + userName.setValue(user["label"]); + } } } }); diff --git a/services/static-webserver/client/source/class/osparc/navigation/NavigationBar.js b/services/static-webserver/client/source/class/osparc/navigation/NavigationBar.js index d3b00b12978..9397aed32e8 100644 --- a/services/static-webserver/client/source/class/osparc/navigation/NavigationBar.js +++ b/services/static-webserver/client/source/class/osparc/navigation/NavigationBar.js @@ -92,16 +92,8 @@ qx.Class.define("osparc.navigation.NavigationBar", { __tabButtons: null, populateLayout: function() { - return new Promise(resolve => { - osparc.data.Resources.get("notifications") - .then(notifications => { - osparc.notification.Notifications.getInstance().addNotifications(notifications); - this.__buildLayout(); - osparc.WindowSizeTracker.getInstance().addListener("changeCompactVersion", () => this.__navBarResized(), this); - resolve(); - }) - .catch(err => console.error(err)); - }); + this.__buildLayout(); + osparc.WindowSizeTracker.getInstance().addListener("changeCompactVersion", () => this.__navBarResized(), this); }, __buildLayout: function() { diff --git a/services/static-webserver/client/source/class/osparc/notification/Notification.js b/services/static-webserver/client/source/class/osparc/notification/Notification.js index 37f102e466f..af219894c0f 100644 --- a/services/static-webserver/client/source/class/osparc/notification/Notification.js +++ b/services/static-webserver/client/source/class/osparc/notification/Notification.js @@ -23,10 +23,12 @@ qx.Class.define("osparc.notification.Notification", { this.set({ id: notificationObj.id, + resourceId: notificationObj.resource_id ? notificationObj.resource_id : null, category: notificationObj.category, actionablePath: notificationObj.actionable_path, title: notificationObj.title, text: notificationObj.text, + userFromId: notificationObj.user_from_id ? notificationObj.user_from_id : null, date: new Date(notificationObj.date), read: ["true", "True", true].includes(notificationObj.read) }); @@ -40,6 +42,13 @@ qx.Class.define("osparc.notification.Notification", { event: "changeId" }, + resourceId: { + check: "String", + init: null, + nullable: true, + event: "changeResourceId" + }, + category: { check: [ "NEW_ORGANIZATION", @@ -74,6 +83,13 @@ qx.Class.define("osparc.notification.Notification", { event: "changeText" }, + userFromId: { + check: "Number", + init: null, + nullable: true, + event: "changeUserFromId" + }, + date: { check: "Date", init: null, diff --git a/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js b/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js index 6ed2ee9356a..b2b002a4f80 100644 --- a/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js +++ b/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js @@ -112,37 +112,102 @@ qx.Class.define("osparc.notification.NotificationUI", { return control || this.base(arguments, id); }, - __applyNotification: function(notification) { - const icon = this.getChildControl("icon"); - notification.bind("category", icon, "source", { - converter: value => { - let source = ""; - switch (value) { - case "NEW_ORGANIZATION": - source = "@FontAwesome5Solid/users/14"; - break; - case "STUDY_SHARED": - source = "@FontAwesome5Solid/file/14"; - break; - case "TEMPLATE_SHARED": - source = "@FontAwesome5Solid/copy/14"; - break; - case "ANNOTATION_NOTE": - source = "@FontAwesome5Solid/file/14"; - break; - case "WALLET_SHARED": - source = "@MaterialIcons/account_balance_wallet/14"; - break; + __applyNotification: async function(notification) { + console.log("notification", notification); + let resourceId = null; + if (notification.getResourceId()) { + resourceId = notification.getResourceId(); + } else if (notification.getActionablePath()) { + // extract it from the actionable path + const actionablePath = notification.getActionablePath(); + resourceId = actionablePath.split("/")[1]; + } + const userFromId = notification.getUserFromId(); + + let source = ""; + let title = ""; + let description = ""; + switch (notification.getCategory()) { + case "NEW_ORGANIZATION": + source = "@FontAwesome5Solid/users/14"; + if (resourceId) { + const group = await osparc.store.Store.getInstance().getGroup(resourceId); + description = "You're now member of '" + group["name"] + "'"; } - return source; - } - }); + break; + case "STUDY_SHARED": + source = "@FontAwesome5Solid/file/14"; + if (resourceId) { + const params = { + url: { + "studyId": resourceId + } + }; + const study = await osparc.data.Resources.getOne("studies", params); + const studyAlias = osparc.product.Utils.getStudyAlias({ + firstUpperCase: true + }); + if (study) { + title = `${studyAlias} '${study["name"]}'`; + } + } + if (userFromId) { + const user = osparc.store.Store.getInstance().getUser(userFromId); + if (user) { + description = "was shared by " + user["label"]; + } + } + break; + case "TEMPLATE_SHARED": + source = "@FontAwesome5Solid/copy/14"; + if (resourceId) { + const template = osparc.store.Store.getInstance().getTemplate(resourceId); + const templateAlias = osparc.product.Utils.getTemplateAlias({ + firstUpperCase: true + }); + if (template) { + title = `${templateAlias} '${template["name"]}'`; + } + } + if (userFromId) { + const user = osparc.store.Store.getInstance().getUser(userFromId); + if (user) { + description = "was shared by " + user["label"]; + } + } + break; + case "ANNOTATION_NOTE": + source = "@FontAwesome5Solid/file/14"; + if (resourceId) { + const params = { + url: { + "studyId": resourceId + } + }; + const study = await osparc.data.Resources.getOne("studies", params); + if (study) { + title = `Note added in '${study["name"]}'`; + } + } + if (userFromId) { + const user = osparc.store.Store.getInstance().getUser(userFromId); + if (user) { + description = "was added by " + user["label"]; + } + } + break; + case "WALLET_SHARED": + source = "@MaterialIcons/account_balance_wallet/14"; + break; + } + const icon = this.getChildControl("icon"); + icon.setSource(source); - const title = this.getChildControl("title"); - notification.bind("title", title, "value"); + const titleLabel = this.getChildControl("title"); + titleLabel.setValue(title ? title : notification.getTitle()); - const text = this.getChildControl("text"); - notification.bind("text", text, "value"); + const descriptionLabel = this.getChildControl("text"); + descriptionLabel.setValue(description ? description : notification.getText()); const date = this.getChildControl("date"); notification.bind("date", date, "value", { @@ -166,23 +231,11 @@ qx.Class.define("osparc.notification.NotificationUI", { } this.fireEvent("notificationTapped"); + osparc.notification.Notifications.markAsRead(notification); + this.__openActionablePath(notification); + }, - if (notification.isRead() === false) { - // set as read - const params = { - url: { - notificationId: notification.getId() - }, - data: { - "read": true - } - }; - osparc.data.Resources.fetch("notifications", "patch", params) - .then(() => notification.setRead(true)) - .catch(() => notification.setRead(false)); - } - - // open actionable path + __openActionablePath: function(notification) { const actionablePath = notification.getActionablePath(); const items = actionablePath.split("/"); const resourceId = items.pop(); diff --git a/services/static-webserver/client/source/class/osparc/notification/Notifications.js b/services/static-webserver/client/source/class/osparc/notification/Notifications.js index 2f7ab34c146..a34a0e2886c 100644 --- a/services/static-webserver/client/source/class/osparc/notification/Notifications.js +++ b/services/static-webserver/client/source/class/osparc/notification/Notifications.js @@ -28,8 +28,9 @@ qx.Class.define("osparc.notification.Notifications", { __newNotificationBase: function(userId) { return { "user_id": userId.toString(), + "user_from_id": osparc.auth.Data.getInstance().getUserId(), "date": new Date().toISOString(), - "product": osparc.product.Utils.getProductName() + "product": osparc.product.Utils.getProductName(), }; }, @@ -38,6 +39,7 @@ qx.Class.define("osparc.notification.Notifications", { const specNotification = { "category": "NEW_ORGANIZATION", "actionable_path": "organization/"+orgId, + "resource_id": orgId, "title": "New organization", "text": "You're now member of a new Organization" }; @@ -55,6 +57,7 @@ qx.Class.define("osparc.notification.Notifications", { const specNotification = { "category": "STUDY_SHARED", "actionable_path": "study/"+studyId, + "resource_id": studyId, "title": `${study} shared`, "text": `A ${study} was shared with you` }; @@ -72,6 +75,7 @@ qx.Class.define("osparc.notification.Notifications", { const specNotification = { "category": "TEMPLATE_SHARED", "actionable_path": "template/"+templateId, + "resource_id": templateId, "title": `${template} shared`, "text": `A ${template} was shared with you` }; @@ -86,6 +90,7 @@ qx.Class.define("osparc.notification.Notifications", { const specNotification = { "category": "ANNOTATION_NOTE", "actionable_path": "study/"+studyId, + "resource_id": studyId, "title": "Note added", "text": "A Note was added for you" }; @@ -100,6 +105,7 @@ qx.Class.define("osparc.notification.Notifications", { const specNotification = { "category": "WALLET_SHARED", "actionable_path": "wallet/"+walletId, + "resource_id": walletId, "title": "Credits shared", "text": "A Credit Account was shared with you" }; @@ -142,7 +148,24 @@ qx.Class.define("osparc.notification.Notifications", { data: this.__newWalletObj(userId, studyId) }; return osparc.data.Resources.fetch("notifications", "post", params); - } + }, + + markAsRead: function(notification) { + if (notification.isRead() === false) { + // set as read + const params = { + url: { + notificationId: notification.getId() + }, + data: { + "read": true + } + }; + osparc.data.Resources.fetch("notifications", "patch", params) + .then(() => notification.setRead(true)) + .catch(() => notification.setRead(false)); + } + }, }, members: { @@ -165,6 +188,14 @@ qx.Class.define("osparc.notification.Notifications", { getNotifications: function() { return this.__notifications; - } + }, + + markAllAsRead: function() { + this.__notifications.forEach(notification => { + if (notification.isRead() === false) { + osparc.notification.Notifications.markAsRead(notification); + } + }); + }, } }); diff --git a/services/static-webserver/client/source/class/osparc/notification/NotificationsButton.js b/services/static-webserver/client/source/class/osparc/notification/NotificationsButton.js index 0b58a2b4e3f..dd8d4b543d4 100644 --- a/services/static-webserver/client/source/class/osparc/notification/NotificationsButton.js +++ b/services/static-webserver/client/source/class/osparc/notification/NotificationsButton.js @@ -82,7 +82,8 @@ qx.Class.define("osparc.notification.NotificationsButton", { }, __updateButton: function() { - const notifications = osparc.notification.Notifications.getInstance().getNotifications(); + const notificationManager = osparc.notification.Notifications.getInstance(); + const notifications = notificationManager.getNotifications(); notifications.forEach(notification => notification.addListener("changeRead", () => this.__updateButton(), this)); const nUnreadNotifications = notifications.filter(notification => notification.getRead() === false).length; @@ -117,6 +118,10 @@ qx.Class.define("osparc.notification.NotificationsButton", { // Show the container this.__notificationsContainer.show(); + // mark all notifications as read + const notificationManager = osparc.notification.Notifications.getInstance(); + notificationManager.markAllAsRead(); + // Add listener for taps outside the container to hide it document.addEventListener("mousedown", this.__onTapOutside.bind(this), true); }, diff --git a/services/static-webserver/client/source/class/osparc/store/Store.js b/services/static-webserver/client/source/class/osparc/store/Store.js index 3a180897e28..0e015ed7811 100644 --- a/services/static-webserver/client/source/class/osparc/store/Store.js +++ b/services/static-webserver/client/source/class/osparc/store/Store.js @@ -427,6 +427,11 @@ qx.Class.define("osparc.store.Store", { } }, + getTemplate: function(templateId) { + const templates = this.getTemplates(); + return templates.find(template => template["uuid"] === templateId); + }, + deleteStudy: function(studyId) { const params = { url: { @@ -626,17 +631,11 @@ qx.Class.define("osparc.store.Store", { }, getUser: function(uid) { - return new Promise(resolve => { - if (uid) { - this.getReachableMembers() - .then(visibleMembers => { - resolve(Object.values(visibleMembers).find(member => member.id === uid)); - }) - .catch(() => resolve(null)); - } else { - resolve(null); - } - }); + if (uid) { + const visibleMembers = this.getReachableMembers(); + return Object.values(visibleMembers).find(member => member.id === uid); + } + return null; }, reloadCreditPrice: function() { diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 82a18aebb3b..a49c71acf17 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -12589,6 +12589,22 @@ components: read: title: Read type: boolean + resource_id: + title: Resource ID + anyOf: + - enum: + - "" + type: string + - type: string + default: "" + user_from_id: + title: User ID of the one creating it + anyOf: + - enum: + - None + type: integer + - type: integer + default: None UserNotificationCreate: title: UserNotificationCreate required: @@ -12628,6 +12644,22 @@ components: type: string - type: string default: UNDEFINED + resource_id: + title: Resource ID + anyOf: + - enum: + - "" + type: string + - type: string + default: "" + user_from_id: + title: User ID of the one creating it + anyOf: + - enum: + - None + type: integer + - type: integer + default: None UserNotificationPatch: title: UserNotificationPatch required: diff --git a/services/web/server/src/simcore_service_webserver/users/_notifications.py b/services/web/server/src/simcore_service_webserver/users/_notifications.py index 256e521f89c..885371f7f65 100644 --- a/services/web/server/src/simcore_service_webserver/users/_notifications.py +++ b/services/web/server/src/simcore_service_webserver/users/_notifications.py @@ -32,6 +32,8 @@ class BaseUserNotification(BaseModel): text: str date: datetime product: Literal["UNDEFINED"] | ProductName = "UNDEFINED" + resource_id: Literal[""] | str = "" + user_from_id: Literal[None] | UserID = None @validator("category", pre=True) @classmethod @@ -106,6 +108,8 @@ class Config: "date": "2023-02-23T16:28:13.122Z", "product": "s4l", "read": False, + "resource_id": "3fb96d89-ff5d-4d27-b5aa-d20d46e20e12", + "user_from_id": "2", }, { "id": "390053c9-3931-40e1-839f-585268f6fd3e", @@ -117,6 +121,8 @@ class Config: "date": "2023-09-29T16:28:13.122Z", "product": "tis", "read": False, + "resource_id": "3fb96d89-ff5d-4d27-b5aa-d20d46e20e13", + "user_from_id": "2", }, ] } diff --git a/services/web/server/tests/unit/isolated/test_user_notifications.py b/services/web/server/tests/unit/isolated/test_user_notifications.py index d606a84297f..7faf71f0aaf 100644 --- a/services/web/server/tests/unit/isolated/test_user_notifications.py +++ b/services/web/server/tests/unit/isolated/test_user_notifications.py @@ -80,6 +80,8 @@ def test_get_notification_key(user_id: UserID): "text": "You're now member of a new Organization", "date": "2023-02-23T16:23:13.122Z", "product": "s4l", + "resource_id": "other_id", + "user_from_id": "2", } ), id="category_from_string", @@ -95,6 +97,8 @@ def test_get_notification_key(user_id: UserID): "text": "You're now member of a new Organization", "date": "2023-02-23T16:23:13.122Z", "product": "tis", + "resource_id": "other_id", + "user_from_id": "2", } ), id="category_from_lower_case_string", diff --git a/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py b/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py index 77aaccade51..0aef84ee328 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py +++ b/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py @@ -196,6 +196,8 @@ async def test_list_user_notifications( "date": "2023-02-23T16:23:13.122Z", "product": "osparc", "read": True, + "resource_id": "3fb96d89-ff5d-4d27-b5aa-d20d46e20e12", + "user_from_id": "2", }, id="with_extra_params_that_will_get_overwritten", ), @@ -265,6 +267,8 @@ async def test_create_user_notification_capped_list_length( "text": "You're now member of a new Organization", "date": "2023-02-23T16:23:13.122Z", "product": "osparc", + "resource_id": "3fb96d89-ff5d-4d27-b5aa-d20d46e20e12", + "user_from_id": "2", }, ) for _ in range(notification_count) From 091c077fbe9b3d4cec8552d0d2672b44932b3f78 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Wed, 6 Nov 2024 10:05:22 +0100 Subject: [PATCH 15/22] =?UTF-8?q?=F0=9F=90=9B=20do=20not=20send=20bps=20me?= =?UTF-8?q?trics=20where=20they=20are=20less=20than=201=20(#6634)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../scheduler/_core/_events_utils.py | 32 ++++++++++++------- .../scheduler/_core/_scheduler.py | 15 +++++---- .../modules/instrumentation/_models.py | 2 +- .../modules/instrumentation/_utils.py | 2 +- .../test_modules_instrumentation__utils.py | 2 +- 5 files changed, 32 insertions(+), 21 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index 9dbe2763bc9..3071cde1060 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -173,9 +173,10 @@ async def service_save_state( size = await sidecars_client.save_service_state( scheduler_data.endpoint, progress_callback=progress_callback ) - get_instrumentation(app).dynamic_sidecar_metrics.push_service_state_rate.labels( - **get_metrics_labels(scheduler_data) - ).observe(get_rate(size, duration.to_flaot())) + if size and size > 0: + get_instrumentation(app).dynamic_sidecar_metrics.push_service_state_rate.labels( + **get_metrics_labels(scheduler_data) + ).observe(get_rate(size, duration.to_float())) await sidecars_client.update_volume_state( scheduler_data.endpoint, @@ -479,10 +480,14 @@ async def _pull_output_ports_with_metrics() -> None: size: int = await sidecars_client.pull_service_output_ports( dynamic_sidecar_endpoint ) - - get_instrumentation(app).dynamic_sidecar_metrics.output_ports_pull_rate.labels( - **get_metrics_labels(scheduler_data) - ).observe(get_rate(size, duration.to_flaot())) + if size and size > 0: + get_instrumentation( + app + ).dynamic_sidecar_metrics.output_ports_pull_rate.labels( + **get_metrics_labels(scheduler_data) + ).observe( + get_rate(size, duration.to_float()) + ) async def _pull_user_services_images_with_metrics() -> None: with track_duration() as duration: @@ -493,16 +498,21 @@ async def _pull_user_services_images_with_metrics() -> None: ).dynamic_sidecar_metrics.pull_user_services_images_duration.labels( **get_metrics_labels(scheduler_data) ).observe( - duration.to_flaot() + duration.to_float() ) async def _restore_service_state_with_metrics() -> None: with track_duration() as duration: size = await sidecars_client.restore_service_state(dynamic_sidecar_endpoint) - get_instrumentation(app).dynamic_sidecar_metrics.pull_service_state_rate.labels( - **get_metrics_labels(scheduler_data) - ).observe(get_rate(size, duration.to_flaot())) + if size and size > 0: + get_instrumentation( + app + ).dynamic_sidecar_metrics.pull_service_state_rate.labels( + **get_metrics_labels(scheduler_data) + ).observe( + get_rate(size, duration.to_float()) + ) tasks = [ _pull_user_services_images_with_metrics(), diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index 1e66fd82c52..b68467a572d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -464,13 +464,14 @@ async def retrieve_service_inputs( ) duration = time.time() - started - get_instrumentation( - self.app - ).dynamic_sidecar_metrics.input_ports_pull_rate.labels( - **get_metrics_labels(scheduler_data) - ).observe( - get_rate(transferred_bytes, duration) - ) + if transferred_bytes and transferred_bytes > 0: + get_instrumentation( + self.app + ).dynamic_sidecar_metrics.input_ports_pull_rate.labels( + **get_metrics_labels(scheduler_data) + ).observe( + get_rate(transferred_bytes, duration) + ) if scheduler_data.restart_policy == RestartPolicy.ON_INPUTS_DOWNLOADED: logger.info("Will restart containers") diff --git a/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_models.py b/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_models.py index 7407885af31..2f1ac9b6548 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_models.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_models.py @@ -88,7 +88,7 @@ def __post_init__(self) -> None: "time to pull docker images", labelnames=_INSTRUMENTATION_LABELS, namespace=_METRICS_NAMESPACE, - buckets=_RATE_BPS_BUCKETS, + buckets=_BUCKETS_TIME_S, subsystem=self.subsystem, registry=self.registry, ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_utils.py index 96b23ae5f1f..2ff4fd8e789 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_utils.py @@ -42,7 +42,7 @@ def set_value(self, value): self._value = float(value) - def to_flaot(self) -> float: + def to_float(self) -> float: if not isinstance(self._value, float): msg = "Value must be a float or an int." raise TypeError(msg) diff --git a/services/director-v2/tests/unit/test_modules_instrumentation__utils.py b/services/director-v2/tests/unit/test_modules_instrumentation__utils.py index 8ebcada1fde..8eab58bdec5 100644 --- a/services/director-v2/tests/unit/test_modules_instrumentation__utils.py +++ b/services/director-v2/tests/unit/test_modules_instrumentation__utils.py @@ -7,4 +7,4 @@ def test_track_duration(): with track_duration() as duration: time.sleep(0.1) - assert duration.to_flaot() > 0.1 + assert duration.to_float() > 0.1 From a515c124d89301967a2a6dca386ddb65b8f0d21f Mon Sep 17 00:00:00 2001 From: Matus Drobuliak <60785969+matusdrobuliak66@users.noreply.github.com> Date: Wed, 6 Nov 2024 13:40:14 +0100 Subject: [PATCH 16/22] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20increase=20timeout?= =?UTF-8?q?=20on=20VTK=20E2E=20test=20(#6677)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/e2e/portal-files/VTK_file.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/portal-files/VTK_file.js b/tests/e2e/portal-files/VTK_file.js index 4e024f8b152..081c5fcdc56 100644 --- a/tests/e2e/portal-files/VTK_file.js +++ b/tests/e2e/portal-files/VTK_file.js @@ -42,7 +42,7 @@ async function runTutorial () { await utils.takeScreenshot(page, screenshotPrefix + 'service_started'); // Some time for setting up service's frontend - await tutorial.waitFor(3000); + await tutorial.waitFor(10000); const frame = await tutorial.getIframe(nodeIdViewer); From 29721841d367dab7a9df4384d0b930318f985844 Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Wed, 6 Nov 2024 15:30:51 +0100 Subject: [PATCH 17/22] =?UTF-8?q?=F0=9F=8E=A8=20[Frontend]=20UX:=20Organiz?= =?UTF-8?q?ation=20member's=20management=20(#6676)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/source/class/osparc/data/Roles.js | 8 ++-- .../desktop/organizations/MembersList.js | 14 ++++-- .../organizations/OrganizationDetails.js | 2 +- .../organizations/OrganizationsList.js | 2 +- .../desktop/organizations/ServicesList.js | 2 +- .../organizations/SharedResourceListItem.js | 47 +++++++++++++++++++ .../desktop/organizations/TemplatesList.js | 2 +- .../osparc/desktop/wallets/MemberListItem.js | 8 ++-- .../osparc/desktop/wallets/WalletListItem.js | 8 ++-- .../osparc/ui/list/CollaboratorListItem.js | 33 ++++++------- .../source/class/osparc/ui/list/ListItem.js | 12 ++--- .../class/osparc/ui/list/ListItemWithMenu.js | 14 +++--- .../osparc/ui/list/OrganizationListItem.js | 6 +++ .../client/source/class/osparc/utils/Utils.js | 6 +-- 14 files changed, 110 insertions(+), 54 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/data/Roles.js b/services/static-webserver/client/source/class/osparc/data/Roles.js index 6320c5a7003..b3a87e6c1c4 100644 --- a/services/static-webserver/client/source/class/osparc/data/Roles.js +++ b/services/static-webserver/client/source/class/osparc/data/Roles.js @@ -24,7 +24,7 @@ qx.Class.define("osparc.data.Roles", { 0: { id: "noRead", label: qx.locale.Manager.tr("Restricted Member"), - longLabel: qx.locale.Manager.tr("Restricted user: no Read access"), + longLabel: qx.locale.Manager.tr("Restricted member: no Read access"), canDo: [ qx.locale.Manager.tr("- can access content shared within the Organization") ] @@ -32,10 +32,10 @@ qx.Class.define("osparc.data.Roles", { 1: { id: "read", label: qx.locale.Manager.tr("Member"), - longLabel: qx.locale.Manager.tr("User: Read access"), + longLabel: qx.locale.Manager.tr("Member: Read access"), canDo: [ - qx.locale.Manager.tr("- can see other users"), - qx.locale.Manager.tr("- can share with other users") + qx.locale.Manager.tr("- can see other members"), + qx.locale.Manager.tr("- can share with other members") ] }, 2: { diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js index 2d065ba1dba..019f40683f0 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js @@ -79,6 +79,7 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { members: { __currentOrg: null, + __introLabel: null, __memberInvitation: null, __membersModel: null, @@ -91,9 +92,7 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { }, __createIntroText: function() { - const msg = this.tr("If you are a manager or administrator, you can add new members and promote or demote existing ones."); - const intro = new qx.ui.basic.Label().set({ - value: msg, + const intro = this.__introLabel = new qx.ui.basic.Label().set({ alignX: "left", rich: true, font: "text-13" @@ -105,7 +104,6 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { const hBox = this.__memberInvitation = new qx.ui.container.Composite(new qx.ui.layout.HBox(10).set({ alignY: "middle" })); - hBox.exclude(); const userEmail = new qx.ui.form.TextField().set({ required: true, @@ -217,8 +215,14 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { const canIWrite = orgModel.getAccessRights().getWrite(); const canIDelete = orgModel.getAccessRights().getDelete(); + + const introText = canIWrite ? + this.tr("You can add new members and promote or demote existing ones.") : + this.tr("You can't add new members to this Organization. Please contact an Administrator or Manager."); + this.__introLabel.setValue(introText); + this.__memberInvitation.set({ - visibility: canIWrite ? "visible" : "excluded" + enabled: canIWrite }); const params = { diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationDetails.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationDetails.js index 83310caa341..6871348d8a0 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationDetails.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationDetails.js @@ -53,7 +53,7 @@ qx.Class.define("osparc.desktop.organizations.OrganizationDetails", { orgModel.bind("thumbnail", organizationListItem, "thumbnail"); orgModel.bind("label", organizationListItem, "title"); orgModel.bind("description", organizationListItem, "subtitle"); - orgModel.bind("nMembers", organizationListItem, "contact"); + orgModel.bind("nMembers", organizationListItem, "role"); orgModel.bind("accessRights", organizationListItem, "accessRights"); // set orgModel to the tab views diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js index 0271fb71dcc..0e205f5d5ad 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js @@ -139,7 +139,7 @@ qx.Class.define("osparc.desktop.organizations.OrganizationsList", { ctrl.bindProperty("thumbnail", "thumbnail", null, item, id); ctrl.bindProperty("label", "title", null, item, id); ctrl.bindProperty("description", "subtitle", null, item, id); - ctrl.bindProperty("nMembers", "contact", null, item, id); + ctrl.bindProperty("nMembers", "role", null, item, id); ctrl.bindProperty("accessRights", "accessRights", null, item, id); }, configureItem: item => { diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/ServicesList.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/ServicesList.js index 9d2d7a6fa1e..bf4f6a95c60 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/ServicesList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/ServicesList.js @@ -70,7 +70,7 @@ qx.Class.define("osparc.desktop.organizations.ServicesList", { const servicesModel = this.__servicesModel = new qx.data.Array(); const servicesCtrl = new qx.data.controller.List(servicesModel, servicesUIList, "name"); servicesCtrl.setDelegate({ - createItem: () => new osparc.desktop.organizations.SharedResourceListItem(), + createItem: () => new osparc.desktop.organizations.SharedResourceListItem("service"), bindItem: (ctrl, item, id) => { ctrl.bindProperty("key", "model", null, item, id); ctrl.bindProperty("key", "key", null, item, id); diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/SharedResourceListItem.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/SharedResourceListItem.js index 4ebe846938c..c9f7d262c32 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/SharedResourceListItem.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/SharedResourceListItem.js @@ -18,6 +18,12 @@ qx.Class.define("osparc.desktop.organizations.SharedResourceListItem", { extend: osparc.ui.list.ListItemWithMenu, + construct: function(resourceType) { + this.__resourceType = resourceType; + + this.base(arguments); + }, + properties: { orgId: { check: "Integer", @@ -38,7 +44,28 @@ qx.Class.define("osparc.desktop.organizations.SharedResourceListItem", { "openMoreInfo": "qx.event.type.Data" }, + statics: { + canDelete: function(accessRights) { + const canDelete = accessRights.getDelete ? accessRights.getDelete() : false; + return canDelete; + }, + + canWrite: function(accessRights) { + let canWrite = accessRights.getWrite ? accessRights.getWrite() : false; + canWrite = canWrite || (accessRights.getWriteAccess ? accessRights.getWriteAccess() : false); + return canWrite; + }, + + canRead: function(accessRights) { + let canRead = accessRights.getRead ? accessRights.getRead() : false; + canRead = canRead || (accessRights.getExecuteAccess ? accessRights.getExecuteAccess() : false); + return canRead; + } + }, + members: { + __resourceType: null, + _createChildControlImpl: function(id) { let control; switch (id) { @@ -63,6 +90,26 @@ qx.Class.define("osparc.desktop.organizations.SharedResourceListItem", { return control || this.base(arguments, id); }, + __getRoleInfo: function(i) { + if (this.__resourceType === "service") { + return osparc.data.Roles.SERVICES[i]; + } + return osparc.data.Roles.STUDY[i]; + }, + + // overridden + _setRole: function() { + const accessRights = this.getAccessRights(); + const role = this.getChildControl("role"); + if (this.self().canDelete(accessRights)) { + role.setValue(this.__getRoleInfo(3).label); + } else if (this.self().canWrite(accessRights)) { + role.setValue(this.__getRoleInfo(2).label); + } else { + role.setValue(this.__getRoleInfo(1).label); + } + }, + // overridden _getInfoButton: function() { const accessRights = this.getAccessRights(); diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/TemplatesList.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/TemplatesList.js index c9e54803c00..70dee6617fc 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/TemplatesList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/TemplatesList.js @@ -70,7 +70,7 @@ qx.Class.define("osparc.desktop.organizations.TemplatesList", { const templatesModel = this.__templatesModel = new qx.data.Array(); const templatesCtrl = new qx.data.controller.List(templatesModel, templatesUIList, "name"); templatesCtrl.setDelegate({ - createItem: () => new osparc.desktop.organizations.SharedResourceListItem(), + createItem: () => new osparc.desktop.organizations.SharedResourceListItem("template"), bindItem: (ctrl, item, id) => { ctrl.bindProperty("uuid", "model", null, item, id); ctrl.bindProperty("uuid", "key", null, item, id); diff --git a/services/static-webserver/client/source/class/osparc/desktop/wallets/MemberListItem.js b/services/static-webserver/client/source/class/osparc/desktop/wallets/MemberListItem.js index 23dc08b7ff9..963126fe3f8 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/wallets/MemberListItem.js +++ b/services/static-webserver/client/source/class/osparc/desktop/wallets/MemberListItem.js @@ -46,13 +46,13 @@ qx.Class.define("osparc.desktop.wallets.MemberListItem", { }, // overridden - _setSubtitle: function() { + _setRole: function() { const accessRights = this.getAccessRights(); - const subtitle = this.getChildControl("contact"); + const role = this.getChildControl("role"); if ("getWrite" in accessRights && accessRights.getWrite()) { - subtitle.setValue(osparc.data.Roles.WALLET[2].label); + role.setValue(osparc.data.Roles.WALLET[2].label); } else if ("getRead" in accessRights && accessRights.getRead()) { - subtitle.setValue(osparc.data.Roles.WALLET[1].label); + role.setValue(osparc.data.Roles.WALLET[1].label); } }, diff --git a/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletListItem.js b/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletListItem.js index 3387638b3de..3e3f0e34e4e 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletListItem.js +++ b/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletListItem.js @@ -272,16 +272,16 @@ qx.Class.define("osparc.desktop.wallets.WalletListItem", { }, // overridden - _setSubtitle: function() { + _setRole: function() { const accessRightss = this.getAccessRights(); const myGid = osparc.auth.Data.getInstance().getGroupId(); const found = accessRightss && accessRightss.find(ar => ar["gid"] === myGid); if (found) { - const subtitle = this.getChildControl("contact"); + const role = this.getChildControl("role"); if (found["write"]) { - subtitle.setValue(osparc.data.Roles.WALLET[2].label); + role.setValue(osparc.data.Roles.WALLET[2].label); } else if (found["read"]) { - subtitle.setValue(osparc.data.Roles.WALLET[1].label); + role.setValue(osparc.data.Roles.WALLET[1].label); } } }, diff --git a/services/static-webserver/client/source/class/osparc/ui/list/CollaboratorListItem.js b/services/static-webserver/client/source/class/osparc/ui/list/CollaboratorListItem.js index 6dd098e5bb3..eb881568b38 100644 --- a/services/static-webserver/client/source/class/osparc/ui/list/CollaboratorListItem.js +++ b/services/static-webserver/client/source/class/osparc/ui/list/CollaboratorListItem.js @@ -18,10 +18,6 @@ qx.Class.define("osparc.ui.list.CollaboratorListItem", { extend: osparc.ui.list.ListItem, - construct: function() { - this.base(arguments); - }, - properties: { collabType: { check: [0, 1, 2], // 0:all, 1:org, 2:user @@ -42,6 +38,7 @@ qx.Class.define("osparc.ui.list.CollaboratorListItem", { event: "changeShowOptions", nullable: true }, + resourceType : { check: "String", event: "changeResourceType", @@ -77,7 +74,7 @@ qx.Class.define("osparc.ui.list.CollaboratorListItem", { }, members: { - _getResource: function(i) { + __getRoleInfo: function(i) { const resource = this.getResourceType(); if (resource === "study" || resource === "template") { return osparc.data.Roles.STUDY[i]; @@ -149,22 +146,22 @@ qx.Class.define("osparc.ui.list.CollaboratorListItem", { return; } - this.__setSubtitle(); + this.__setRole(); const menu = this.__getOptionsMenu(); const optionsMenu = this.getChildControl("options"); optionsMenu.setMenu(menu); }, - __setSubtitle: function() { + __setRole: function() { const accessRights = this.getAccessRights(); - const subtitle = this.getChildControl("contact"); + const role = this.getChildControl("role"); if (this.self().canDelete(accessRights)) { - subtitle.setValue(this._getResource(3).label); + role.setValue(this.__getRoleInfo(3).label); } else if (this.self().canWrite(accessRights)) { - subtitle.setValue(this._getResource(2).label); + role.setValue(this.__getRoleInfo(2).label); } else { - subtitle.setValue(this._getResource(1).label); + role.setValue(this.__getRoleInfo(1).label); } }, @@ -174,17 +171,17 @@ qx.Class.define("osparc.ui.list.CollaboratorListItem", { }); const accessRights = this.getAccessRights(); - let currentRole = this._getResource(1); + let currentRole = this.__getRoleInfo(1); if (this.self().canDelete(accessRights)) { - currentRole = this._getResource(3); + currentRole = this.__getRoleInfo(3); } else if (this.self().canWrite(accessRights)) { - currentRole = this._getResource(2); + currentRole = this.__getRoleInfo(2); } // promote/demote actions switch (currentRole.id) { case "read": { - const promoteButton = new qx.ui.menu.Button(this.tr(`Promote to ${this._getResource(2).label}`)); + const promoteButton = new qx.ui.menu.Button(this.tr(`Promote to ${this.__getRoleInfo(2).label}`)); promoteButton.addListener("execute", () => { this.fireDataEvent("promoteToEditor", { gid: this.getKey(), @@ -196,7 +193,7 @@ qx.Class.define("osparc.ui.list.CollaboratorListItem", { } case "write": { const resource = this.getResourceType(); - const promoteButton = new qx.ui.menu.Button(this.tr(`Promote to ${this._getResource(3).label}`)); + const promoteButton = new qx.ui.menu.Button(this.tr(`Promote to ${this.__getRoleInfo(3).label}`)); promoteButton.setVisibility(resource === "service" ? "excluded" : "visible"); promoteButton.addListener("execute", () => { this.fireDataEvent("promoteToOwner", { @@ -205,7 +202,7 @@ qx.Class.define("osparc.ui.list.CollaboratorListItem", { }); }); menu.add(promoteButton); - const demoteButton = new qx.ui.menu.Button(this.tr(`Demote to ${this._getResource(1).label}`)); + const demoteButton = new qx.ui.menu.Button(this.tr(`Demote to ${this.__getRoleInfo(1).label}`)); demoteButton.addListener("execute", () => { this.fireDataEvent("demoteToUser", { gid: this.getKey(), @@ -216,7 +213,7 @@ qx.Class.define("osparc.ui.list.CollaboratorListItem", { break; } case "delete": { - const demoteButton = new qx.ui.menu.Button(this.tr(`Demote to ${this._getResource(2).label}`)); + const demoteButton = new qx.ui.menu.Button(this.tr(`Demote to ${this.__getRoleInfo(2).label}`)); demoteButton.addListener("execute", () => { this.fireDataEvent("demoteToEditor", { gid: this.getKey(), diff --git a/services/static-webserver/client/source/class/osparc/ui/list/ListItem.js b/services/static-webserver/client/source/class/osparc/ui/list/ListItem.js index 70031dc0f9a..e046f420805 100644 --- a/services/static-webserver/client/source/class/osparc/ui/list/ListItem.js +++ b/services/static-webserver/client/source/class/osparc/ui/list/ListItem.js @@ -34,7 +34,7 @@ * c.bindProperty("thumbnail", "thumbnail", null, item, id); * c.bindProperty("name", "title", null, item, id); * c.bindProperty("description", "subtitle", null, item, id); - * c.bindProperty("contact", "contact", null, item, id); + * c.bindProperty("role", "role", null, item, id); * } * }); * @@ -98,9 +98,9 @@ qx.Class.define("osparc.ui.list.ListItem", { nullable : true }, - contact: { + role: { check : "String", - apply : "__applyContact", + apply : "__applyRole", nullable : true } }, @@ -180,7 +180,7 @@ qx.Class.define("osparc.ui.list.ListItem", { column: 1 }); break; - case "contact": + case "role": control = new qx.ui.basic.Label().set({ font: "text-13", alignY: "middle" @@ -244,11 +244,11 @@ qx.Class.define("osparc.ui.list.ListItem", { label.setValue(value); }, - __applyContact: function(value) { + __applyRole: function(value) { if (value === null) { return; } - const label = this.getChildControl("contact"); + const label = this.getChildControl("role"); label.setValue(value); }, diff --git a/services/static-webserver/client/source/class/osparc/ui/list/ListItemWithMenu.js b/services/static-webserver/client/source/class/osparc/ui/list/ListItemWithMenu.js index 22e65916d08..4d9d1d054bb 100644 --- a/services/static-webserver/client/source/class/osparc/ui/list/ListItemWithMenu.js +++ b/services/static-webserver/client/source/class/osparc/ui/list/ListItemWithMenu.js @@ -80,27 +80,29 @@ qx.Class.define("osparc.ui.list.ListItemWithMenu", { return; } + this._setRole(); + this._getInfoButton(); this.__applyOptions(); }, - _setSubtitle: function() { + _setRole: function() { const accessRights = this.getAccessRights(); - const subtitle = this.getChildControl("contact"); + const role = this.getChildControl("role"); if ( "getDelete" in accessRights && accessRights.getDelete() ) { - subtitle.setValue(osparc.data.Roles.ORG[3].label); + role.setValue(osparc.data.Roles.ORG[3].label); } else if ("getWrite" in accessRights && accessRights.getWrite()) { - subtitle.setValue(osparc.data.Roles.ORG[2].label); + role.setValue(osparc.data.Roles.ORG[2].label); } else if ( ("getRead" in accessRights && accessRights.getRead()) || ("getExecute" in accessRights && accessRights.getExecute()) ) { - subtitle.setValue(osparc.data.Roles.ORG[1].label); + role.setValue(osparc.data.Roles.ORG[1].label); } else { - subtitle.setValue(osparc.data.Roles.ORG[0].label); + role.setValue(osparc.data.Roles.ORG[0].label); } }, diff --git a/services/static-webserver/client/source/class/osparc/ui/list/OrganizationListItem.js b/services/static-webserver/client/source/class/osparc/ui/list/OrganizationListItem.js index 905e520f2f8..6da8294c011 100644 --- a/services/static-webserver/client/source/class/osparc/ui/list/OrganizationListItem.js +++ b/services/static-webserver/client/source/class/osparc/ui/list/OrganizationListItem.js @@ -33,6 +33,12 @@ qx.Class.define("osparc.ui.list.OrganizationListItem", { }, members: { + // overridden + _setRole: function() { + // Role field was already filled up with the nMembers + return; + }, + // overridden _getOptionsMenu: function() { let menu = null; diff --git a/services/static-webserver/client/source/class/osparc/utils/Utils.js b/services/static-webserver/client/source/class/osparc/utils/Utils.js index 629468c1312..5c751c2ee8f 100644 --- a/services/static-webserver/client/source/class/osparc/utils/Utils.js +++ b/services/static-webserver/client/source/class/osparc/utils/Utils.js @@ -1023,20 +1023,20 @@ qx.Class.define("osparc.utils.Utils", { }, setIdToWidget: (qWidget, id) => { - if (qWidget.getContentElement) { + if (qWidget.getContentElement && qWidget.getContentElement()) { qWidget.getContentElement().setAttribute("osparc-test-id", id); } }, getIdFromWidget: qWidget => { - if (qWidget.getContentElement) { + if (qWidget.getContentElement && qWidget.getContentElement()) { return qWidget.getContentElement().getAttribute("osparc-test-id"); } return null; }, setKeyToWidget: (qWidget, id) => { - if (qWidget.getContentElement) { + if (qWidget.getContentElement && qWidget.getContentElement()) { qWidget.getContentElement().setAttribute("osparc-test-key", id); } }, From 96350ac9d548dcb8d515e82d285cb10b7c2051b5 Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Wed, 6 Nov 2024 16:26:12 +0100 Subject: [PATCH 18/22] =?UTF-8?q?=F0=9F=8E=A8=F0=9F=90=9B=20[Frontend]=20N?= =?UTF-8?q?otifications:=20disable=20unknown=20resources=20(#6679)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../osparc/notification/NotificationUI.js | 73 +++++++++---------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js b/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js index b2b002a4f80..b8071b8d3ef 100644 --- a/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js +++ b/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js @@ -112,8 +112,7 @@ qx.Class.define("osparc.notification.NotificationUI", { return control || this.base(arguments, id); }, - __applyNotification: async function(notification) { - console.log("notification", notification); + __applyNotification: function(notification) { let resourceId = null; if (notification.getResourceId()) { resourceId = notification.getResourceId(); @@ -124,90 +123,88 @@ qx.Class.define("osparc.notification.NotificationUI", { } const userFromId = notification.getUserFromId(); - let source = ""; - let title = ""; - let description = ""; + const icon = this.getChildControl("icon"); + const titleLabel = this.getChildControl("title"); + titleLabel.setValue(notification.getTitle()); + const descriptionLabel = this.getChildControl("text"); + descriptionLabel.setValue(notification.getText()); + switch (notification.getCategory()) { case "NEW_ORGANIZATION": - source = "@FontAwesome5Solid/users/14"; + icon.setSource("@FontAwesome5Solid/users/14"); if (resourceId) { - const group = await osparc.store.Store.getInstance().getGroup(resourceId); - description = "You're now member of '" + group["name"] + "'"; + osparc.store.Store.getInstance().getGroup(resourceId) + .then(group => descriptionLabel.setValue("You're now member of '" + group["label"] + "'")) + .catch(() => this.setEnabled(false)); } break; case "STUDY_SHARED": - source = "@FontAwesome5Solid/file/14"; + icon.setSource("@FontAwesome5Solid/file/14"); if (resourceId) { const params = { url: { "studyId": resourceId } }; - const study = await osparc.data.Resources.getOne("studies", params); - const studyAlias = osparc.product.Utils.getStudyAlias({ - firstUpperCase: true - }); - if (study) { - title = `${studyAlias} '${study["name"]}'`; - } + osparc.data.Resources.getOne("studies", params) + .then(study => { + const studyAlias = osparc.product.Utils.getStudyAlias({ + firstUpperCase: true + }); + titleLabel.setValue(`${studyAlias} '${study["name"]}'`); + }) + .catch(() => this.setEnabled(false)); } if (userFromId) { const user = osparc.store.Store.getInstance().getUser(userFromId); if (user) { - description = "was shared by " + user["label"]; + descriptionLabel.setValue("was shared by " + user["label"]); } } break; case "TEMPLATE_SHARED": - source = "@FontAwesome5Solid/copy/14"; + icon.setSource("@FontAwesome5Solid/copy/14"); if (resourceId) { const template = osparc.store.Store.getInstance().getTemplate(resourceId); - const templateAlias = osparc.product.Utils.getTemplateAlias({ - firstUpperCase: true - }); if (template) { - title = `${templateAlias} '${template["name"]}'`; + const templateAlias = osparc.product.Utils.getTemplateAlias({ + firstUpperCase: true + }); + titleLabel.setValue(`${templateAlias} '${template["name"]}'`); + } else { + this.setEnabled(false); } } if (userFromId) { const user = osparc.store.Store.getInstance().getUser(userFromId); if (user) { - description = "was shared by " + user["label"]; + descriptionLabel.setValue("was shared by " + user["label"]); } } break; case "ANNOTATION_NOTE": - source = "@FontAwesome5Solid/file/14"; + icon.setSource("@FontAwesome5Solid/file/14"); if (resourceId) { const params = { url: { "studyId": resourceId } }; - const study = await osparc.data.Resources.getOne("studies", params); - if (study) { - title = `Note added in '${study["name"]}'`; - } + osparc.data.Resources.getOne("studies", params) + .then(study => titleLabel.setValue(`Note added in '${study["name"]}'`)) + .catch(() => this.setEnabled(false)); } if (userFromId) { const user = osparc.store.Store.getInstance().getUser(userFromId); if (user) { - description = "was added by " + user["label"]; + descriptionLabel.setValue("was added by " + user["label"]); } } break; case "WALLET_SHARED": - source = "@MaterialIcons/account_balance_wallet/14"; + icon.setSource("@MaterialIcons/account_balance_wallet/14"); break; } - const icon = this.getChildControl("icon"); - icon.setSource(source); - - const titleLabel = this.getChildControl("title"); - titleLabel.setValue(title ? title : notification.getTitle()); - - const descriptionLabel = this.getChildControl("text"); - descriptionLabel.setValue(description ? description : notification.getText()); const date = this.getChildControl("date"); notification.bind("date", date, "value", { From 4abb9f23e99c5cd854e348a81f8e8e4e4c88c691 Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Wed, 6 Nov 2024 17:29:25 +0100 Subject: [PATCH 19/22] =?UTF-8?q?=F0=9F=8E=A8=20[Frontend]=20Meaningful=20?= =?UTF-8?q?captions=20on=20Confirmation=20windows=20(#6680)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../source/class/osparc/dashboard/FolderButtonItem.js | 5 +++-- .../client/source/class/osparc/dashboard/ListButtonBase.js | 4 ++-- .../client/source/class/osparc/dashboard/StudyBrowser.js | 6 ++++-- .../source/class/osparc/dashboard/StudyBrowserHeader.js | 4 ++-- .../client/source/class/osparc/dashboard/TemplateBrowser.js | 6 ++++-- .../source/class/osparc/dashboard/WorkspaceButtonItem.js | 1 + .../class/osparc/dashboard/WorkspacesAndFoldersTreeItem.js | 5 +---- .../client/source/class/osparc/data/model/Node.js | 1 + .../client/source/class/osparc/desktop/SlideshowView.js | 1 + .../client/source/class/osparc/desktop/StudyEditor.js | 1 + .../client/source/class/osparc/desktop/WorkbenchView.js | 2 ++ .../class/osparc/desktop/organizations/MembersList.js | 1 + .../class/osparc/desktop/organizations/OrganizationsList.js | 1 + .../osparc/desktop/paymentMethods/PaymentMethodListItem.js | 1 + .../class/osparc/desktop/preferences/pages/ClustersPage.js | 1 + .../class/osparc/desktop/preferences/pages/TokensPage.js | 2 ++ .../source/class/osparc/notification/NotificationUI.js | 1 + .../client/source/class/osparc/share/Collaborators.js | 6 ++++-- .../client/source/class/osparc/share/CollaboratorsStudy.js | 1 + .../source/class/osparc/share/CollaboratorsWorkspace.js | 1 + .../client/source/class/osparc/task/TaskUI.js | 1 + 21 files changed, 36 insertions(+), 16 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/dashboard/FolderButtonItem.js b/services/static-webserver/client/source/class/osparc/dashboard/FolderButtonItem.js index 355b1db7c87..526f7032c27 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/FolderButtonItem.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/FolderButtonItem.js @@ -197,7 +197,7 @@ qx.Class.define("osparc.dashboard.FolderButtonItem", { menu.addSeparator(); const deleteButton = new qx.ui.menu.Button(this.tr("Delete"), "@FontAwesome5Solid/trash/12"); - deleteButton.addListener("execute", () => this.__deleteStudyRequested(), this); + deleteButton.addListener("execute", () => this.__deleteFolderRequested(), this); menu.add(deleteButton); menuButton.setMenu(menu); @@ -237,9 +237,10 @@ qx.Class.define("osparc.dashboard.FolderButtonItem", { folderEditor.addListener("cancel", () => win.close()); }, - __deleteStudyRequested: function() { + __deleteFolderRequested: function() { const msg = this.tr("Are you sure you want to delete") + " " + this.getTitle() + "?"; const confirmationWin = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete Folder"), confirmText: this.tr("Delete"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/ListButtonBase.js b/services/static-webserver/client/source/class/osparc/dashboard/ListButtonBase.js index 099a9d0488c..86decb00157 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/ListButtonBase.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/ListButtonBase.js @@ -51,8 +51,8 @@ qx.Class.define("osparc.dashboard.ListButtonBase", { PROGRESS: 4, TAGS: 5, ICONS_LAYOUT: 6, - SHARED: 7, - OWNER: 8, + OWNER: 7, + SHARED: 8, LAST_CHANGE: 9, TSR: 10, HITS: 11, diff --git a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js index ab29d814808..7349d7d46b5 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js @@ -455,6 +455,7 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { __showMoveToWorkspaceWarningMessage: function() { const msg = this.tr("The permissions will be taken from the new workspace."); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Move"), confirmText: this.tr("Move"), }); win.open(); @@ -1724,9 +1725,10 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { __createConfirmWindow: function(studyNames) { const rUSure = this.tr("Are you sure you want to delete"); - const studiesText = osparc.product.Utils.getStudyAlias({plural: true}); - const msg = rUSure + (studyNames.length > 1 ? ` ${studyNames.length} ${studiesText} ?` : ` ${studyNames[0]}?`) + const studyAlias = osparc.product.Utils.getStudyAlias({plural: studyNames.length > 1}); + const msg = rUSure + (studyNames.length > 1 ? ` ${studyNames.length} ${studyAlias}?` : ` ${studyNames[0]}?`) const confirmationWin = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete") + " " + studyAlias, confirmText: this.tr("Delete"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js index 36cb6375e26..9e2ca51b434 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js @@ -108,7 +108,7 @@ qx.Class.define("osparc.dashboard.StudyBrowserHeader", { }); this._add(control); break; - case "workspace-title": + case "title": control = new qx.ui.basic.Label().set({ font: "text-16", alignY: "middle", @@ -197,7 +197,7 @@ qx.Class.define("osparc.dashboard.StudyBrowserHeader", { __buildLayout: function() { this.getChildControl("icon"); - const title = this.getChildControl("workspace-title"); + const title = this.getChildControl("title"); title.resetCursor(); title.removeListener("tap", this.__titleTapped, this); this.getChildControl("breadcrumbs"); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/TemplateBrowser.js b/services/static-webserver/client/source/class/osparc/dashboard/TemplateBrowser.js index 2b200676688..d597d8a438c 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/TemplateBrowser.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/TemplateBrowser.js @@ -193,9 +193,10 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { }); updateAllButton.exclude(); updateAllButton.addListener("tap", () => { - const templatesText = osparc.product.Utils.getTemplateAlias({plural: true}); - const msg = this.tr("Are you sure you want to update all ") + templatesText + "?"; + const templatesAlias = osparc.product.Utils.getTemplateAlias({plural: true}); + const msg = this.tr("Are you sure you want to update all ") + templatesAlias + "?"; const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Update") + " " + templatesAlias, confirmText: this.tr("Update all"), confirmAction: "create" }); @@ -327,6 +328,7 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { const rUSure = this.tr("Are you sure you want to delete "); const msg = rUSure + "" + templateData.name + "?"; const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete"), confirmText: this.tr("Delete"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonItem.js b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonItem.js index dd7b80e5044..5581ec3212b 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonItem.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonItem.js @@ -250,6 +250,7 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonItem", { let msg = this.tr("Are you sure you want to delete") + " " + this.getTitle() + "?"; msg += "
" + this.tr("All the content of the workspace will be deleted."); const confirmationWin = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete Workspace"), confirmText: this.tr("Delete"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/WorkspacesAndFoldersTreeItem.js b/services/static-webserver/client/source/class/osparc/dashboard/WorkspacesAndFoldersTreeItem.js index 071ba7e3d6d..ccedd3b3b3d 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/WorkspacesAndFoldersTreeItem.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/WorkspacesAndFoldersTreeItem.js @@ -24,10 +24,7 @@ qx.Class.define("osparc.dashboard.WorkspacesAndFoldersTreeItem", { this.set({ indent: 12, // defaults to 19, decorator: "rounded", - }); - - this.set({ - maxWidth: osparc.dashboard.ResourceBrowserBase.SIDE_SPACER_WIDTH - 12 + maxWidth: osparc.dashboard.ResourceBrowserBase.SIDE_SPACER_WIDTH - 12, }); this.setNotHoveredStyle(); diff --git a/services/static-webserver/client/source/class/osparc/data/model/Node.js b/services/static-webserver/client/source/class/osparc/data/model/Node.js index b52c08dbf25..0a1be2d9ae8 100644 --- a/services/static-webserver/client/source/class/osparc/data/model/Node.js +++ b/services/static-webserver/client/source/class/osparc/data/model/Node.js @@ -949,6 +949,7 @@ qx.Class.define("osparc.data.model.Node", { if (preferencesSettings.getConfirmStopNode()) { const msg = this.tr("Do you really want Stop and Save the current state?"); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Stop"), confirmText: this.tr("Stop") }); win.center(); diff --git a/services/static-webserver/client/source/class/osparc/desktop/SlideshowView.js b/services/static-webserver/client/source/class/osparc/desktop/SlideshowView.js index e4817393152..593088bc4cd 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/SlideshowView.js +++ b/services/static-webserver/client/source/class/osparc/desktop/SlideshowView.js @@ -456,6 +456,7 @@ qx.Class.define("osparc.desktop.SlideshowView", { if (preferencesSettings.getConfirmDeleteNode()) { const msg = this.tr("Are you sure you want to delete node?"); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete"), confirmText: this.tr("Delete"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/StudyEditor.js b/services/static-webserver/client/source/class/osparc/desktop/StudyEditor.js index 774eb0d508e..393fc842796 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/StudyEditor.js +++ b/services/static-webserver/client/source/class/osparc/desktop/StudyEditor.js @@ -598,6 +598,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { this.getStudyLogger().info(null, "The pipeline is up-to-date"); const msg = this.tr("The pipeline is up-to-date. Do you want to re-run it?"); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Re-run"), confirmText: this.tr("Run"), confirmAction: "create" }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/WorkbenchView.js b/services/static-webserver/client/source/class/osparc/desktop/WorkbenchView.js index ca51e61093b..1daeea1c0f1 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/static-webserver/client/source/class/osparc/desktop/WorkbenchView.js @@ -1119,6 +1119,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { if (!avoidConfirmation && preferencesSettings.getConfirmDeleteNode()) { const msg = this.tr("Are you sure you want to delete the selected node?"); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete Node"), confirmText: this.tr("Delete"), confirmAction: "delete" }); @@ -1140,6 +1141,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { if (preferencesSettings.getConfirmDeleteNode()) { const msg = this.tr("Are you sure you want to delete the selected ") + nodeIds.length + " nodes?"; const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete Nodes"), confirmText: this.tr("Delete"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js index 019f40683f0..090c1ce705c 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js @@ -571,6 +571,7 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { } rUSure += "

" + this.tr("If you Leave, the page will be reloaded."); const confirmationWin = new osparc.ui.window.Confirmation(rUSure).set({ + caption: this.tr("Leave Organization"), confirmText: this.tr("Leave"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js index 0e205f5d5ad..705e943ef5a 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js @@ -238,6 +238,7 @@ qx.Class.define("osparc.desktop.organizations.OrganizationsList", { const name = org.getLabel(); const msg = this.tr("Are you sure you want to delete ") + name + "?"; const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete Organization"), confirmText: this.tr("Delete"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodListItem.js b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodListItem.js index 61cb2b28f21..9342f10f066 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodListItem.js +++ b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodListItem.js @@ -184,6 +184,7 @@ qx.Class.define("osparc.desktop.paymentMethods.PaymentMethodListItem", { __deletePressed: function() { const msg = this.tr("Are you sure you want to delete the Payment Method?"); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete Payment Method"), confirmText: this.tr("Delete"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ClustersPage.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ClustersPage.js index 447333712eb..be0a0eed354 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ClustersPage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ClustersPage.js @@ -352,6 +352,7 @@ qx.Class.define("osparc.desktop.preferences.pages.ClustersPage", { const name = cluster.getName(); const msg = this.tr("Are you sure you want to delete ") + name + "?"; const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete Cluster"), confirmText: this.tr("Delete"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TokensPage.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TokensPage.js index cd19c6f73df..ddc9d94f60f 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TokensPage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TokensPage.js @@ -144,6 +144,7 @@ qx.Class.define("osparc.desktop.preferences.pages.TokensPage", { const msg = this.tr("Do you want to delete the API key?"); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete API key"), confirmText: this.tr("Delete"), confirmAction: "delete" }); @@ -284,6 +285,7 @@ qx.Class.define("osparc.desktop.preferences.pages.TokensPage", { const msg = this.tr("Do you want to delete the Token?"); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete Token"), confirmText: this.tr("Delete"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js b/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js index b8071b8d3ef..da49db7f0a4 100644 --- a/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js +++ b/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js @@ -305,6 +305,7 @@ qx.Class.define("osparc.notification.NotificationUI", { if (myAccountWindow.openWallets()) { const msg = this.tr("Do you want to make it the default Credit Account?"); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Default Credit Account"), confirmAction: "create" }); win.center(); diff --git a/services/static-webserver/client/source/class/osparc/share/Collaborators.js b/services/static-webserver/client/source/class/osparc/share/Collaborators.js index 6bde9a32e86..66e75202ce6 100644 --- a/services/static-webserver/client/source/class/osparc/share/Collaborators.js +++ b/services/static-webserver/client/source/class/osparc/share/Collaborators.js @@ -357,9 +357,10 @@ qx.Class.define("osparc.share.Collaborators", { // check also user is not "prjOwner". Backend will silently not let the frontend remove that user. (this._serializedDataCopy["prjOwner"] !== osparc.auth.Data.getInstance().getEmail()) ) { - const leaveButton = new qx.ui.form.Button(this.tr("Leave") + " " + osparc.product.Utils.getStudyAlias({ + const leaveText = this.tr("Leave") + " " + osparc.product.Utils.getStudyAlias({ firstUpperCase: true - })).set({ + }); + const leaveButton = new qx.ui.form.Button(leaveText).set({ allowGrowX: false, visibility: Object.keys(this._serializedDataCopy["accessRights"]).includes(myGid.toString()) ? "visible" : "excluded" }); @@ -370,6 +371,7 @@ qx.Class.define("osparc.share.Collaborators", { msg += this.tr("If you remove yourself, there won't be any other Owners."); } const win = new osparc.ui.window.Confirmation(msg).set({ + caption: leaveText, confirmText: this.tr("Leave"), confirmAction: "delete" }); diff --git a/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js b/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js index 80ecbe75006..e6b03e5818e 100644 --- a/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js +++ b/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js @@ -226,6 +226,7 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { if (isOrganization) { const msg = this.tr(`Demoting to ${osparc.data.Roles.STUDY[1].label} will remove write access to all the members of the Organization. Are you sure?`); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Demote"), confirmAction: "delete", confirmText: this.tr("Yes") }); diff --git a/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js b/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js index 63c2a33b9a9..b1c3791e4a7 100644 --- a/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js +++ b/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js @@ -162,6 +162,7 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { if (isOrganization) { const msg = this.tr(`Demoting to ${osparc.data.Roles.WORKSPACE[1].label} will remove write access to all the members of the Organization. Are you sure?`); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Demote"), confirmAction: "delete", confirmText: this.tr("Yes") }); diff --git a/services/static-webserver/client/source/class/osparc/task/TaskUI.js b/services/static-webserver/client/source/class/osparc/task/TaskUI.js index 6fe3a952ec9..513bf34ec91 100644 --- a/services/static-webserver/client/source/class/osparc/task/TaskUI.js +++ b/services/static-webserver/client/source/class/osparc/task/TaskUI.js @@ -118,6 +118,7 @@ qx.Class.define("osparc.task.TaskUI", { stopButton.addListener("tap", () => { const msg = this.tr("Are you sure you want to cancel the task?"); const win = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Cancel Task"), confirmText: this.tr("Cancel"), confirmAction: "delete" }); From 2b6b0a9a0c28a6d0e72499686311790f93e743d9 Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:52:08 +0100 Subject: [PATCH 20/22] =?UTF-8?q?=F0=9F=8E=A8=20[Frontend]=20Rephrase=20sh?= =?UTF-8?q?aring=20messages=20(#6683)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../desktop/organizations/MembersList.js | 26 ++++++++--------- .../desktop/preferences/pages/ClustersPage.js | 4 +-- .../osparc/node/UpdateResourceLimitsView.js | 2 +- .../osparc/share/CollaboratorsService.js | 18 +++++------- .../class/osparc/share/CollaboratorsStudy.js | 29 ++++++++++--------- .../osparc/share/CollaboratorsWorkspace.js | 26 ++++++++--------- 6 files changed, 53 insertions(+), 52 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js index 090c1ce705c..e5cb935cdf8 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js @@ -322,7 +322,7 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { }; osparc.data.Resources.fetch("organizationMembers", "post", params) .then(() => { - const text = orgMemberEmail + this.tr(" successfully added."); + const text = orgMemberEmail + this.tr(" successfully added"); if (productEveryone && productEveryone["gid"] === parseInt(orgId)) { // demote the new member to user const params2 = { @@ -379,12 +379,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { }; osparc.data.Resources.fetch("organizationMembers", "patch", params) .then(() => { - osparc.FlashMessenger.getInstance().logAs(orgMember["name"] + this.tr(` successfully promoted to ${osparc.data.Roles.ORG[1].label}`)); + osparc.FlashMessenger.getInstance().logAs(this.tr(`Successfully promoted to ${osparc.data.Roles.ORG[1].label}`)); osparc.store.Store.getInstance().reset("organizationMembers"); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong promoting ") + orgMember["name"], "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong promoting to ") + osparc.data.Roles.ORG[1].label, "ERROR"); console.error(err); }); }, @@ -406,14 +406,14 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { osparc.data.Resources.fetch("organizationMembers", "patch", params) .then(() => { if (msg === undefined) { - msg = orgMember["name"] + this.tr(` successfully demoted to ${osparc.data.Roles.ORG[0].label}`); + msg = this.tr(`Successfully demoted to ${osparc.data.Roles.ORG[0].label}`); } osparc.FlashMessenger.getInstance().logAs(msg); osparc.store.Store.getInstance().reset("organizationMembers"); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong demoting ") + orgMember["name"], "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong demoting to ") + osparc.data.Roles.ORG[0].label, "ERROR"); console.error(err); }); }, @@ -434,12 +434,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { }; osparc.data.Resources.fetch("organizationMembers", "patch", params) .then(() => { - osparc.FlashMessenger.getInstance().logAs(orgMember["name"] + this.tr(` successfully promoted to ${osparc.data.Roles.ORG[2].label}`)); + osparc.FlashMessenger.getInstance().logAs(this.tr(`Successfully promoted to ${osparc.data.Roles.ORG[2].label}`)); osparc.store.Store.getInstance().reset("organizationMembers"); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong promoting ") + orgMember["name"], "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong promoting to ") + osparc.data.Roles.ORG[2].label, "ERROR"); console.error(err); }); }, @@ -460,12 +460,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { }; osparc.data.Resources.fetch("organizationMembers", "patch", params) .then(() => { - osparc.FlashMessenger.getInstance().logAs(orgMember["name"] + this.tr(` successfully promoted to ${osparc.data.Roles.ORG[3].label}`)); + osparc.FlashMessenger.getInstance().logAs(this.tr(`Successfully promoted to ${osparc.data.Roles.ORG[3].label}`)); osparc.store.Store.getInstance().reset("organizationMembers"); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong promoting ") + orgMember["name"], "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong promoting to ") + osparc.data.Roles.ORG[3].label, "ERROR"); console.error(err); }); }, @@ -486,12 +486,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { }; osparc.data.Resources.fetch("organizationMembers", "patch", params) .then(() => { - osparc.FlashMessenger.getInstance().logAs(orgMember["name"] + this.tr(` successfully demoted to ${osparc.data.Roles.ORG[1].label}`)); + osparc.FlashMessenger.getInstance().logAs(this.tr(`Successfully demoted to ${osparc.data.Roles.ORG[1].label}`)); osparc.store.Store.getInstance().reset("organizationMembers"); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong demoting ") + orgMember["name"], "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong demoting to ") + osparc.data.Roles.ORG[1].label, "ERROR"); console.error(err); }); }, @@ -512,12 +512,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { }; osparc.data.Resources.fetch("organizationMembers", "patch", params) .then(() => { - osparc.FlashMessenger.getInstance().logAs(orgMember["name"] + this.tr(` successfully demoted to ${osparc.data.Roles.ORG[3].label}`)); + osparc.FlashMessenger.getInstance().logAs(this.tr(`Successfully demoted to ${osparc.data.Roles.ORG[3].label}`)); osparc.store.Store.getInstance().reset("organizationMembers"); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong demoting ") + orgMember["name"], "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong demoting to ") + osparc.data.Roles.ORG[3].label, "ERROR"); console.error(err); }); }, diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ClustersPage.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ClustersPage.js index be0a0eed354..e68f7f7577f 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ClustersPage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ClustersPage.js @@ -488,12 +488,12 @@ qx.Class.define("osparc.desktop.preferences.pages.ClustersPage", { }; osparc.data.Resources.fetch("clusters", "patch", params) .then(() => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Member(s) added")); + osparc.FlashMessenger.getInstance().logAs(this.tr("Cluster successfully shared")); osparc.store.Store.getInstance().reset("clusters"); this.__reloadClusters(true); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong with the invitation"), "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong sharing the Cluster"), "ERROR"); console.error(err); }); }, diff --git a/services/static-webserver/client/source/class/osparc/node/UpdateResourceLimitsView.js b/services/static-webserver/client/source/class/osparc/node/UpdateResourceLimitsView.js index 9179633ac82..f6770a7e675 100644 --- a/services/static-webserver/client/source/class/osparc/node/UpdateResourceLimitsView.js +++ b/services/static-webserver/client/source/class/osparc/node/UpdateResourceLimitsView.js @@ -92,7 +92,7 @@ qx.Class.define("osparc.node.UpdateResourceLimitsView", { value = osparc.utils.Utils.bytesToGiB(value); } const spinner = new qx.ui.form.Spinner(0, value, 512).set({ - singleStep: 0.1 + singleStep: 1.0 }); const nf = new qx.util.format.NumberFormat(); nf.setMinimumFractionDigits(2); diff --git a/services/static-webserver/client/source/class/osparc/share/CollaboratorsService.js b/services/static-webserver/client/source/class/osparc/share/CollaboratorsService.js index 4f540ebb211..5ae55925f58 100644 --- a/services/static-webserver/client/source/class/osparc/share/CollaboratorsService.js +++ b/services/static-webserver/client/source/class/osparc/share/CollaboratorsService.js @@ -93,15 +93,13 @@ qx.Class.define("osparc.share.CollaboratorsService", { osparc.store.Services.patchServiceData(this._serializedDataCopy, "accessRights", newAccessRights) .then(() => { this.fireDataEvent("updateAccessRights", this._serializedDataCopy); - let text = this.tr("Editor(s) successfully added."); - text += "
"; - text += this.tr("The user will not get notified."); + const text = this.tr("Service successfully shared"); osparc.FlashMessenger.getInstance().logAs(text); this._reloadCollaboratorsList(); }) .catch(err => { console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong adding editor(s)"), "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong sharing the Service"), "ERROR"); }); }, @@ -122,12 +120,12 @@ qx.Class.define("osparc.share.CollaboratorsService", { osparc.store.Services.patchServiceData(this._serializedDataCopy, "accessRights", this._serializedDataCopy["accessRights"]) .then(() => { this.fireDataEvent("updateAccessRights", this._serializedDataCopy); - osparc.FlashMessenger.getInstance().logAs(this.tr("Member successfully removed")); + osparc.FlashMessenger.getInstance().logAs(collaborator["name"] + this.tr(" successfully removed")); this._reloadCollaboratorsList(); }) .catch(err => { console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing Member"), "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing ") + collaborator["name"], "ERROR"); }) .finally(() => { if (item) { @@ -156,8 +154,8 @@ qx.Class.define("osparc.share.CollaboratorsService", { this.__make( collaborator["gid"], this.self().getOwnerAccessRight(), - this.tr("Viewer successfully made Editor"), - this.tr("Something went wrong making Viewer Editor"), + this.tr(`Successfully promoted to ${osparc.data.Roles.SERVICE[2].label}`), + this.tr(`Something went wrong promoting to ${osparc.data.Roles.SERVICE[2].label}`), item ); }, @@ -170,8 +168,8 @@ qx.Class.define("osparc.share.CollaboratorsService", { this.__make( collaborator["gid"], this.self().getCollaboratorAccessRight(), - this.tr("Editor successfully made Viewer"), - this.tr("Something went wrong making Editor Viewer"), + this.tr(`Successfully demoted to ${osparc.data.Roles.SERVICE[1].label}`), + this.tr(`Something went wrong demoting ${osparc.data.Roles.SERVICE[1].label}`), item ); }, diff --git a/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js b/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js index e6b03e5818e..58f47059cd6 100644 --- a/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js +++ b/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js @@ -127,15 +127,18 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { return; } + const resourceAlias = this._resourceType === "template" ? + osparc.product.Utils.getTemplateAlias({firstUpperCase: true}) : + osparc.product.Utils.getStudyAlias({firstUpperCase: true}); const newCollaborators = {}; gids.forEach(gid => { newCollaborators[gid] = this._resourceType === "study" ? this.self().getCollaboratorAccessRight() : this.self().getViewerAccessRight(); }); osparc.info.StudyUtils.addCollaborators(this._serializedDataCopy, newCollaborators) .then(() => { - this.fireDataEvent("updateAccessRights", this._serializedDataCopy); - const text = this.tr("User(s) successfully added."); + const text = resourceAlias + this.tr(" successfully shared"); osparc.FlashMessenger.getInstance().logAs(text); + this.fireDataEvent("updateAccessRights", this._serializedDataCopy); this._reloadCollaboratorsList(); this.__pushNotifications(gids); @@ -143,7 +146,7 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { }) .catch(err => { console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went adding user(s)"), "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong sharing the ") + resourceAlias, "ERROR"); }); }, @@ -155,12 +158,12 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { return osparc.info.StudyUtils.removeCollaborator(this._serializedDataCopy, collaborator["gid"]) .then(() => { this.fireDataEvent("updateAccessRights", this._serializedDataCopy); - osparc.FlashMessenger.getInstance().logAs(this.tr("Member successfully removed")); + osparc.FlashMessenger.getInstance().logAs(collaborator["name"] + this.tr(" successfully removed")); this._reloadCollaboratorsList(); }) .catch(err => { console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing Member"), "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing ") + collaborator["name"], "ERROR"); }) .finally(() => { if (item) { @@ -193,8 +196,8 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { this.__make( collaborator["gid"], this.self().getCollaboratorAccessRight(), - this.tr(`${osparc.data.Roles.STUDY[1].label} successfully changed ${osparc.data.Roles.STUDY[2].label}`), - this.tr(`Something went wrong changing ${osparc.data.Roles.STUDY[1].label} to ${osparc.data.Roles.STUDY[2].label}`), + this.tr(`Successfully promoted to ${osparc.data.Roles.STUDY[2].label}`), + this.tr(`Something went wrong promoting to ${osparc.data.Roles.STUDY[2].label}`), item ); }, @@ -203,8 +206,8 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { this.__make( collaborator["gid"], this.self().getOwnerAccessRight(), - this.tr(`${osparc.data.Roles.STUDY[2].label} successfully changed to ${osparc.data.Roles.STUDY[3].label}`), - this.tr(`Something went wrong changing ${osparc.data.Roles.STUDY[2].label} to ${osparc.data.Roles.STUDY[3].label}`), + this.tr(`Successfully promoted to ${osparc.data.Roles.STUDY[3].label}`), + this.tr(`Something went wrong promoting to ${osparc.data.Roles.STUDY[3].label}`), item ); }, @@ -215,8 +218,8 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { this.__make( gid, this.self().getViewerAccessRight(), - this.tr(`${osparc.data.Roles.STUDY[2].label} successfully changed to ${osparc.data.Roles.STUDY[1].label}`), - this.tr(`Something went wrong changing ${osparc.data.Roles.STUDY[2].label} to ${osparc.data.Roles.STUDY[1].label}`), + this.tr(`Successfully demoted to ${osparc.data.Roles.STUDY[1].label}`), + this.tr(`Something went wrong demoting to ${osparc.data.Roles.STUDY[1].label}`), itm ); }; @@ -246,8 +249,8 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { this.__make( collaborator["gid"], this.self().getCollaboratorAccessRight(), - this.tr(`${osparc.data.Roles.STUDY[3].label} successfully changed to ${osparc.data.Roles.STUDY[2].label}`), - this.tr(`Something went wrong changing ${osparc.data.Roles.STUDY[3].label} to ${osparc.data.Roles.STUDY[2].label}`), + this.tr(`Successfully demoted to ${osparc.data.Roles.STUDY[2].label}`), + this.tr(`Something went wrong demoting to ${osparc.data.Roles.STUDY[2].label}`), item ); }, diff --git a/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js b/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js index b1c3791e4a7..d8bb906794c 100644 --- a/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js +++ b/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js @@ -72,14 +72,14 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { gids.forEach(gid => newCollaborators[gid] = this.self().getCollaboratorAccessRight()); osparc.store.Workspaces.getInstance().addCollaborators(this.__workspace.getWorkspaceId(), newCollaborators) .then(() => { - this.fireDataEvent("updateAccessRights", this.__workspace.serialize()); - const text = this.tr("User(s) successfully added."); + const text = this.tr("Workspace successfully shared"); osparc.FlashMessenger.getInstance().logAs(text); + this.fireDataEvent("updateAccessRights", this.__workspace.serialize()); this._reloadCollaboratorsList(); }) .catch(err => { console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went adding user(s)"), "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong sharing the Workspace"), "ERROR"); }); }, @@ -91,12 +91,12 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { osparc.store.Workspaces.getInstance().removeCollaborator(this.__workspace.getWorkspaceId(), collaborator["gid"]) .then(() => { this.fireDataEvent("updateAccessRights", this.__workspace.serialize()); - osparc.FlashMessenger.getInstance().logAs(this.tr("Member successfully removed")); + osparc.FlashMessenger.getInstance().logAs(collaborator["name"] + this.tr(" successfully removed")); this._reloadCollaboratorsList(); }) .catch(err => { console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing Member"), "ERROR"); + osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing ") + collaborator["name"], "ERROR"); }) .finally(() => { if (item) { @@ -129,8 +129,8 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { this.__make( collaborator["gid"], this.self().getCollaboratorAccessRight(), - this.tr(`${osparc.data.Roles.WORKSPACE[1].label} successfully changed ${osparc.data.Roles.WORKSPACE[2].label}`), - this.tr(`Something went wrong changing ${osparc.data.Roles.WORKSPACE[1].label} to ${osparc.data.Roles.WORKSPACE[2].label}`), + this.tr(`Successfully promoted to ${osparc.data.Roles.WORKSPACE[2].label}`), + this.tr(`Something went wrong promoting to ${osparc.data.Roles.WORKSPACE[2].label}`), item ); }, @@ -139,8 +139,8 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { this.__make( collaborator["gid"], this.self().getOwnerAccessRight(), - this.tr(`${osparc.data.Roles.WORKSPACE[2].label} successfully changed to ${osparc.data.Roles.WORKSPACE[3].label}`), - this.tr(`Something went wrong changing ${osparc.data.Roles.WORKSPACE[2].label} to ${osparc.data.Roles.WORKSPACE[3].label}`), + this.tr(`Successfully promoted to ${osparc.data.Roles.WORKSPACE[3].label}`), + this.tr(`Something went wrong promoting to ${osparc.data.Roles.WORKSPACE[3].label}`), item ); }, @@ -151,8 +151,8 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { this.__make( gid, this.self().getViewerAccessRight(), - this.tr(`${osparc.data.Roles.WORKSPACE[2].label} successfully changed to ${osparc.data.Roles.WORKSPACE[1].label}`), - this.tr(`Something went wrong changing ${osparc.data.Roles.WORKSPACE[2].label} to ${osparc.data.Roles.WORKSPACE[1].label}`), + this.tr(`Successfully demoted to ${osparc.data.Roles.WORKSPACE[1].label}`), + this.tr(`Something went wrong demoting to ${osparc.data.Roles.WORKSPACE[1].label}`), itm ); }; @@ -182,8 +182,8 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { this.__make( collaborator["gid"], this.self().getCollaboratorAccessRight(), - this.tr(`${osparc.data.Roles.WORKSPACE[3].label} successfully changed to ${osparc.data.Roles.WORKSPACE[2].label}`), - this.tr(`Something went wrong changing ${osparc.data.Roles.WORKSPACE[3].label} to ${osparc.data.Roles.WORKSPACE[2].label}`), + this.tr(`Successfully demoted to ${osparc.data.Roles.WORKSPACE[2].label}`), + this.tr(`Something went wrong demoting to ${osparc.data.Roles.WORKSPACE[2].label}`), item ); } From d904673c38321aab11681ce8c213def6575c5bf6 Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:02:52 +0100 Subject: [PATCH 21/22] =?UTF-8?q?=F0=9F=90=9B=20[Frontend]=20Highlight=20d?= =?UTF-8?q?efault=20Pricing=20Unit=20(#6685)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/source/class/osparc/study/PricingUnits.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/study/PricingUnits.js b/services/static-webserver/client/source/class/osparc/study/PricingUnits.js index 5238825a838..02597e76760 100644 --- a/services/static-webserver/client/source/class/osparc/study/PricingUnits.js +++ b/services/static-webserver/client/source/class/osparc/study/PricingUnits.js @@ -58,13 +58,13 @@ qx.Class.define("osparc.study.PricingUnits", { if (preselectedPricingUnit) { const buttonFound = buttons.find(button => button.getUnitData().getPricingUnitId() === preselectedPricingUnit["pricingUnitId"]); if (buttonFound) { - buttonFound.execute(); + buttonFound.setValue(true); } } else { // preselect default buttons.forEach(button => { if (button.getUnitData().isDefault()) { - button.execute(); + button.setValue(true); } }); } From 927319c428d340e6d203d0374c96b7a1d7303b97 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Fri, 8 Nov 2024 07:16:48 +0100 Subject: [PATCH 22/22] =?UTF-8?q?=F0=9F=90=9B=20Refactor=20storage=20setup?= =?UTF-8?q?=20functions=20to=20avoid=20errors=20(#6686)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../storage/src/simcore_service_storage/s3.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/services/storage/src/simcore_service_storage/s3.py b/services/storage/src/simcore_service_storage/s3.py index 8814db3757a..f5e07f5c6ea 100644 --- a/services/storage/src/simcore_service_storage/s3.py +++ b/services/storage/src/simcore_service_storage/s3.py @@ -21,12 +21,13 @@ async def setup_s3_client(app) -> AsyncGenerator[None, None]: - with log_context(log, logging.DEBUG, msg=f"setup {__name__}.setup.cleanup_ctx"): + client = None + + with log_context(log, logging.DEBUG, msg="setup.s3_client.cleanup_ctx"): storage_settings: Settings = app[APP_CONFIG_KEY] storage_s3_settings = storage_settings.STORAGE_S3 assert storage_s3_settings # nosec - client = None async for attempt in AsyncRetrying( wait=wait_fixed(RETRY_WAIT_SECS), before_sleep=before_sleep_log(log, logging.WARNING), @@ -45,17 +46,21 @@ async def setup_s3_client(app) -> AsyncGenerator[None, None]: assert client # nosec app[APP_S3_KEY] = client - yield - # tear-down + yield + + with log_context(log, logging.DEBUG, msg="teardown.s3_client.cleanup_ctx"): + if client: await client.close() async def setup_s3_bucket(app: web.Application): - storage_s3_settings = app[APP_CONFIG_KEY].STORAGE_S3 - client = get_s3_client(app) - await client.create_bucket( - bucket=storage_s3_settings.S3_BUCKET_NAME, region=storage_s3_settings.S3_REGION - ) + with log_context(log, logging.DEBUG, msg="setup.s3_bucket.cleanup_ctx"): + storage_s3_settings = app[APP_CONFIG_KEY].STORAGE_S3 + client = get_s3_client(app) + await client.create_bucket( + bucket=storage_s3_settings.S3_BUCKET_NAME, + region=storage_s3_settings.S3_REGION, + ) yield