From 00b094474968496f4dc65eb5ffa69f3131b45cce Mon Sep 17 00:00:00 2001 From: Mads Bisgaard <126242332+bisgaard-itis@users.noreply.github.com> Date: Mon, 3 Jun 2024 15:06:49 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=A8=20propagate=20job=20parent=20ids?= =?UTF-8?q?=20through=20api=20server=20(#5903)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/api-server/openapi.json | 60 +++++++++++++++++++ .../api/routes/solvers_jobs.py | 11 +++- .../api/routes/studies.py | 11 +++- .../api/routes/studies_jobs.py | 13 +++- .../services/webserver.py | 35 ++++++++++- .../test_api_routers_solvers_jobs_delete.py | 35 ++++++++++- .../api_studies/test_api_routes_studies.py | 44 +++++++++++++- .../test_api_routes_studies_jobs.py | 29 +++++++++ 8 files changed, 225 insertions(+), 13 deletions(-) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 828876780ce..7989d4a9c48 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -2038,6 +2038,26 @@ }, "name": "hidden", "in": "query" + }, + { + "required": false, + "schema": { + "type": "string", + "format": "uuid", + "title": "X-Simcore-Parent-Project-Uuid" + }, + "name": "x-simcore-parent-project-uuid", + "in": "header" + }, + { + "required": false, + "schema": { + "type": "string", + "format": "uuid", + "title": "X-Simcore-Parent-Node-Id" + }, + "name": "x-simcore-parent-node-id", + "in": "header" } ], "requestBody": { @@ -3254,6 +3274,26 @@ }, "name": "study_id", "in": "path" + }, + { + "required": false, + "schema": { + "type": "string", + "format": "uuid", + "title": "X-Simcore-Parent-Project-Uuid" + }, + "name": "x-simcore-parent-project-uuid", + "in": "header" + }, + { + "required": false, + "schema": { + "type": "string", + "format": "uuid", + "title": "X-Simcore-Parent-Node-Id" + }, + "name": "x-simcore-parent-node-id", + "in": "header" } ], "responses": { @@ -3382,6 +3422,26 @@ }, "name": "hidden", "in": "query" + }, + { + "required": false, + "schema": { + "type": "string", + "format": "uuid", + "title": "X-Simcore-Parent-Project-Uuid" + }, + "name": "x-simcore-parent-project-uuid", + "in": "header" + }, + { + "required": false, + "schema": { + "type": "string", + "format": "uuid", + "title": "X-Simcore-Parent-Node-Id" + }, + "name": "x-simcore-parent-node-id", + "in": "header" } ], "requestBody": { diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 9fe37ada46b..82a461d91e7 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -4,9 +4,11 @@ from collections.abc import Callable from typing import Annotated, Any -from fastapi import APIRouter, Depends, Query, Request, status +from fastapi import APIRouter, Depends, Header, Query, Request, status from models_library.api_schemas_webserver.projects import ProjectCreateNew, ProjectGet from models_library.clusters import ClusterID +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID from pydantic.types import PositiveInt from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES @@ -87,6 +89,8 @@ async def create_job( url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], hidden: Annotated[bool, Query()] = True, + x_simcore_parent_project_uuid: Annotated[ProjectID | None, Header()] = None, + x_simcore_parent_node_id: Annotated[NodeID | None, Header()] = None, ): """Creates a job in a specific release with given inputs. @@ -107,7 +111,10 @@ async def create_job( project_in: ProjectCreateNew = create_new_project_for_job(solver, pre_job, inputs) new_project: ProjectGet = await webserver_api.create_project( - project_in, is_hidden=hidden + project_in, + is_hidden=hidden, + parent_project_uuid=x_simcore_parent_project_uuid, + parent_node_id=x_simcore_parent_node_id, ) assert new_project # nosec assert new_project.uuid == pre_job.id # nosec diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies.py b/services/api-server/src/simcore_service_api_server/api/routes/studies.py index 29327098aed..d4c37bb0512 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies.py @@ -1,9 +1,11 @@ import logging from typing import Annotated, Final -from fastapi import APIRouter, Depends, status +from fastapi import APIRouter, Depends, Header, status from fastapi_pagination.api import create_page from models_library.api_schemas_webserver.projects import ProjectGet +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID from ...models.pagination import OnePage, Page, PaginationParams from ...models.schemas.errors import ErrorGet @@ -85,9 +87,14 @@ async def get_study( async def clone_study( study_id: StudyID, webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + x_simcore_parent_project_uuid: Annotated[ProjectID | None, Header()] = None, + x_simcore_parent_node_id: Annotated[NodeID | None, Header()] = None, ): project: ProjectGet = await webserver_api.clone_project( - project_id=study_id, hidden=False + project_id=study_id, + hidden=False, + parent_project_uuid=x_simcore_parent_project_uuid, + parent_node_id=x_simcore_parent_node_id, ) return _create_study_from_project(project) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index 68be168b217..e320214867d 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -2,13 +2,15 @@ from collections.abc import Callable from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, status +from fastapi import APIRouter, Depends, Header, Query, Request, status from fastapi.responses import RedirectResponse from models_library.api_schemas_webserver.projects import ProjectName, ProjectPatch from models_library.api_schemas_webserver.projects_nodes import NodeOutputs from models_library.clusters import ClusterID from models_library.function_services_catalog.services import file_picker +from models_library.projects import ProjectID from models_library.projects_nodes import InputID, InputTypes +from models_library.projects_nodes_io import NodeID from pydantic import PositiveInt from servicelib.logging_utils import log_context @@ -81,11 +83,18 @@ async def create_study_job( webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], hidden: Annotated[bool, Query()] = True, + x_simcore_parent_project_uuid: ProjectID | None = Header(default=None), + x_simcore_parent_node_id: NodeID | None = Header(default=None), ) -> Job: """ hidden -- if True (default) hides project from UI """ - project = await webserver_api.clone_project(project_id=study_id, hidden=hidden) + project = await webserver_api.clone_project( + project_id=study_id, + hidden=hidden, + parent_project_uuid=x_simcore_parent_project_uuid, + parent_node_id=x_simcore_parent_node_id, + ) job = create_job_from_study( study_key=study_id, project=project, job_inputs=job_inputs ) diff --git a/services/api-server/src/simcore_service_api_server/services/webserver.py b/services/api-server/src/simcore_service_api_server/services/webserver.py index 89a0b78fc14..ee29d81f582 100644 --- a/services/api-server/src/simcore_service_api_server/services/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services/webserver.py @@ -46,6 +46,10 @@ from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import PositiveInt from servicelib.aiohttp.long_running_tasks.server import TaskStatus +from servicelib.common_headers import ( + X_SIMCORE_PARENT_NODE_ID, + X_SIMCORE_PARENT_PROJECT_UUID, +) from tenacity import TryAgain from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log @@ -253,12 +257,22 @@ async def update_me(self, profile_update: ProfileUpdate) -> Profile: @_exception_mapper({}) async def create_project( - self, project: ProjectCreateNew, *, is_hidden: bool + self, + project: ProjectCreateNew, + *, + is_hidden: bool, + parent_project_uuid: ProjectID | None, + parent_node_id: NodeID | None, ) -> ProjectGet: # POST /projects --> 202 Accepted + _headers = { + X_SIMCORE_PARENT_PROJECT_UUID: parent_project_uuid, + X_SIMCORE_PARENT_NODE_ID: parent_node_id, + } response = await self.client.post( "/projects", params={"hidden": is_hidden}, + headers={k: f"{v}" for k, v in _headers.items() if v is not None}, json=jsonable_encoder(project, by_alias=True, exclude={"state"}), cookies=self.session_cookies, ) @@ -267,10 +281,25 @@ async def create_project( return ProjectGet.parse_obj(result) @_exception_mapper(_JOB_STATUS_MAP) - async def clone_project(self, *, project_id: UUID, hidden: bool) -> ProjectGet: + async def clone_project( + self, + *, + project_id: UUID, + hidden: bool, + parent_project_uuid: ProjectID | None, + parent_node_id: NodeID | None, + ) -> ProjectGet: query = {"from_study": project_id, "hidden": hidden} + _headers = { + X_SIMCORE_PARENT_PROJECT_UUID: parent_project_uuid, + X_SIMCORE_PARENT_NODE_ID: parent_node_id, + } + response = await self.client.post( - "/projects", cookies=self.session_cookies, params=query + "/projects", + cookies=self.session_cookies, + params=query, + headers={k: f"{v}" for k, v in _headers.items() if v is not None}, ) response.raise_for_status() result = await self._wait_for_long_running_task_results(response) diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py index 9a98a4ac4fa..5e61c9d1b82 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import TypedDict +from uuid import UUID import httpx import jinja2 @@ -13,9 +14,15 @@ from pydantic import parse_file_as from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter +from servicelib.common_headers import ( + X_SIMCORE_PARENT_NODE_ID, + X_SIMCORE_PARENT_PROJECT_UUID, +) from simcore_service_api_server.models.schemas.jobs import Job, JobInputs from starlette import status +_faker = Faker() + class MockedBackendApiDict(TypedDict): catalog: MockRouter | None @@ -158,6 +165,10 @@ async def test_create_and_delete_solver_job( # Run a job and delete when finished +@pytest.mark.parametrize( + "parent_node_id, parent_project_id", + [(_faker.uuid4(), _faker.uuid4()), (None, None)], +) @pytest.mark.parametrize("hidden", [True, False]) async def test_create_job( auth: httpx.BasicAuth, @@ -166,29 +177,47 @@ async def test_create_job( solver_version: str, mocked_backend_services_apis_for_create_and_delete_solver_job: MockedBackendApiDict, hidden: bool, + parent_project_id: UUID | None, + parent_node_id: UUID | None, ): mock_webserver_router = ( mocked_backend_services_apis_for_create_and_delete_solver_job["webserver"] ) + assert mock_webserver_router is not None callback = mock_webserver_router["create_projects"].side_effect + assert callback is not None def create_project_side_effect(request: httpx.Request): + # check `hidden` bool query = dict(elm.split("=") for elm in request.url.query.decode().split("&")) _hidden = query.get("hidden") assert _hidden == ("true" if hidden else "false") + + # check parent project and node id + if parent_project_id is not None: + assert f"{parent_project_id}" == dict(request.headers).get( + X_SIMCORE_PARENT_PROJECT_UUID.lower() + ) + if parent_node_id is not None: + assert f"{parent_node_id}" == dict(request.headers).get( + X_SIMCORE_PARENT_NODE_ID.lower() + ) return callback(request) - mock_webserver_router = ( - mocked_backend_services_apis_for_create_and_delete_solver_job["webserver"] - ) mock_webserver_router["create_projects"].side_effect = create_project_side_effect # create Job + header_dict = {} + if parent_project_id is not None: + header_dict[X_SIMCORE_PARENT_PROJECT_UUID] = f"{parent_project_id}" + if parent_node_id is not None: + header_dict[X_SIMCORE_PARENT_NODE_ID] = f"{parent_node_id}" resp = await client.post( f"/v0/solvers/{solver_key}/releases/{solver_version}/jobs", auth=auth, params={"hidden": f"{hidden}"}, + headers=header_dict, json=JobInputs( values={ "x": 3.14, diff --git a/services/api-server/tests/unit/api_studies/test_api_routes_studies.py b/services/api-server/tests/unit/api_studies/test_api_routes_studies.py index 3090184a962..858787953b7 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routes_studies.py +++ b/services/api-server/tests/unit/api_studies/test_api_routes_studies.py @@ -6,6 +6,7 @@ from collections.abc import Callable from pathlib import Path from typing import Any, TypedDict +from uuid import UUID import httpx import pytest @@ -14,9 +15,15 @@ from pydantic import parse_file_as, parse_obj_as from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter +from servicelib.common_headers import ( + X_SIMCORE_PARENT_NODE_ID, + X_SIMCORE_PARENT_PROJECT_UUID, +) from simcore_service_api_server.models.schemas.errors import ErrorGet from simcore_service_api_server.models.schemas.studies import Study, StudyID, StudyPort +_faker = Faker() + class MockedBackendApiDict(TypedDict): catalog: MockRouter | None @@ -137,17 +144,52 @@ async def test_list_study_ports( @pytest.mark.acceptance_test( "Implements https://github.com/ITISFoundation/osparc-simcore/issues/4651" ) +@pytest.mark.parametrize( + "parent_node_id, parent_project_id", + [(_faker.uuid4(), _faker.uuid4()), (None, None)], +) async def test_clone_study( client: httpx.AsyncClient, auth: httpx.BasicAuth, study_id: StudyID, mocked_webserver_service_api_base: MockRouter, patch_webserver_long_running_project_tasks: Callable[[MockRouter], MockRouter], + parent_project_id: UUID | None, + parent_node_id: UUID | None, ): # Mocks /projects patch_webserver_long_running_project_tasks(mocked_webserver_service_api_base) - resp = await client.post(f"/v0/studies/{study_id}:clone", auth=auth) + callback = mocked_webserver_service_api_base["create_projects"].side_effect + assert callback is not None + + def clone_project_side_effect(request: httpx.Request): + if parent_project_id is not None: + _parent_project_id = dict(request.headers).get( + X_SIMCORE_PARENT_PROJECT_UUID.lower() + ) + assert _parent_project_id == f"{parent_project_id}" + if parent_node_id is not None: + _parent_node_id = dict(request.headers).get( + X_SIMCORE_PARENT_NODE_ID.lower() + ) + assert _parent_node_id == f"{parent_node_id}" + return callback(request) + + mocked_webserver_service_api_base[ + "create_projects" + ].side_effect = clone_project_side_effect + + _headers = {} + if parent_project_id is not None: + _headers[X_SIMCORE_PARENT_PROJECT_UUID] = f"{parent_project_id}" + if parent_node_id is not None: + _headers[X_SIMCORE_PARENT_NODE_ID] = f"{parent_node_id}" + resp = await client.post( + f"/v0/studies/{study_id}:clone", headers=_headers, auth=auth + ) + + assert mocked_webserver_service_api_base["create_projects"].called assert resp.status_code == status.HTTP_201_CREATED diff --git a/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py b/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py index 736ead2acea..443fa943548 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py +++ b/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py @@ -20,10 +20,16 @@ HttpApiCallCaptureModel, ) from respx import MockRouter +from servicelib.common_headers import ( + X_SIMCORE_PARENT_NODE_ID, + X_SIMCORE_PARENT_PROJECT_UUID, +) from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.models.schemas.jobs import Job, JobOutputs from simcore_service_api_server.models.schemas.studies import Study, StudyID +_faker = Faker() + @pytest.mark.xfail(reason="Still not implemented") @pytest.mark.acceptance_test( @@ -188,6 +194,10 @@ def _check_response(response: httpx.Response, status_code: int): _check_response(response, status.HTTP_204_NO_CONTENT) +@pytest.mark.parametrize( + "parent_node_id, parent_project_id", + [(_faker.uuid4(), _faker.uuid4()), (None, None)], +) @pytest.mark.parametrize("hidden", [True, False]) async def test_create_study_job( client: httpx.AsyncClient, @@ -198,6 +208,8 @@ async def test_create_study_job( project_tests_dir: Path, fake_study_id: UUID, hidden: bool, + parent_project_id: UUID | None, + parent_node_id: UUID | None, ): _capture_file: Final[Path] = project_tests_dir / "mocks" / "create_study_job.json" @@ -216,12 +228,23 @@ def _default_side_effect( assert project_id is not None assert project_id in name if capture.method == "POST": + # test hidden boolean _default_side_effect.post_called = True query_dict = dict( elm.split("=") for elm in request.url.query.decode().split("&") ) _hidden = query_dict.get("hidden") assert _hidden == ("true" if hidden else "false") + + # test parent project and node ids + if parent_project_id is not None: + assert f"{parent_project_id}" == dict(request.headers).get( + X_SIMCORE_PARENT_PROJECT_UUID.lower() + ) + if parent_node_id is not None: + assert f"{parent_node_id}" == dict(request.headers).get( + X_SIMCORE_PARENT_NODE_ID.lower() + ) return capture.response_body _default_side_effect.patch_called = False @@ -236,9 +259,15 @@ def _default_side_effect( side_effects_callbacks=[_default_side_effect] * 5, ) + header_dict = {} + if parent_project_id is not None: + header_dict[X_SIMCORE_PARENT_PROJECT_UUID] = f"{parent_project_id}" + if parent_node_id is not None: + header_dict[X_SIMCORE_PARENT_NODE_ID] = f"{parent_node_id}" response = await client.post( f"{API_VTAG}/studies/{fake_study_id}/jobs", auth=auth, + headers=header_dict, params={"hidden": f"{hidden}"}, json={"values": {}}, )