From d38a6c361545fae272c49266a89f806a3b8719ea Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 <60785969+matusdrobuliak66@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:31:51 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=A8=20improve=20project=20full=20searc?= =?UTF-8?q?h=20(#6483)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/specs/web-server/_projects_crud.py | 9 ++- .../servicelib/aiohttp/requests_validation.py | 3 + .../api/v0/openapi.yaml | 70 ++++++++++++++++- .../projects/_crud_api_read.py | 37 ++++++++- .../projects/_crud_handlers.py | 19 +++-- .../projects/_crud_handlers_models.py | 40 +++++++++- .../projects/_db_utils.py | 4 +- .../simcore_service_webserver/projects/db.py | 75 ++++++++++++++---- .../projects/exceptions.py | 4 + ...t_workspaces__list_projects_full_search.py | 76 +++++++++++++++++++ 10 files changed, 306 insertions(+), 31 deletions(-) diff --git a/api/specs/web-server/_projects_crud.py b/api/specs/web-server/_projects_crud.py index 1c76ff4230d..46073f921fc 100644 --- a/api/specs/web-server/_projects_crud.py +++ b/api/specs/web-server/_projects_crud.py @@ -142,10 +142,17 @@ async def clone_project( @router.get( "/projects:search", - response_model=Page[ProjectListItem], + response_model=Page[ProjectListFullSearchParams], ) async def list_projects_full_search( _params: Annotated[ProjectListFullSearchParams, Depends()], + order_by: Annotated[ + Json, + Query( + description="Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date) and direction (asc|desc). The default sorting order is ascending.", + example='{"field": "last_change_date", "direction": "desc"}', + ), + ] = ('{"field": "last_change_date", "direction": "desc"}',), ): ... diff --git a/packages/service-library/src/servicelib/aiohttp/requests_validation.py b/packages/service-library/src/servicelib/aiohttp/requests_validation.py index 2fd5d0e41f0..085243c5d26 100644 --- a/packages/service-library/src/servicelib/aiohttp/requests_validation.py +++ b/packages/service-library/src/servicelib/aiohttp/requests_validation.py @@ -166,7 +166,10 @@ def parse_request_query_parameters_as( resource_name=request.rel_url.path, use_error_v1=use_enveloped_error_v1, ): + # NOTE: Currently, this does not take into consideration cases where there are multiple + # query parameters with the same key. However, we are not using such cases anywhere at the moment. data = dict(request.query) + if hasattr(parameters_schema_cls, "parse_obj"): return parameters_schema_cls.parse_obj(data) model: ModelClass = parse_obj_as(parameters_schema_cls, data) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 13aa1b47dad..2fcba19b92c 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -3298,6 +3298,18 @@ paths: summary: List Projects Full Search operationId: list_projects_full_search parameters: + - description: Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date) + and direction (asc|desc). The default sorting order is ascending. + required: false + schema: + title: Order By + description: Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date) + and direction (asc|desc). The default sorting order is ascending. + default: + - '{"field": "last_change_date", "direction": "desc"}' + example: '{"field": "last_change_date", "direction": "desc"}' + name: order_by + in: query - required: false schema: title: Limit @@ -3323,13 +3335,19 @@ paths: type: string name: text in: query + - required: false + schema: + title: Tag Ids + type: string + name: tag_ids + in: query responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/Page_ProjectListItem_' + $ref: '#/components/schemas/Page_ProjectListFullSearchParams_' /v0/projects/{project_id}/inactivity: get: tags: @@ -9555,6 +9573,25 @@ components: $ref: '#/components/schemas/ProjectIterationResultItem' additionalProperties: false description: Paginated response model of ItemTs + Page_ProjectListFullSearchParams_: + title: Page[ProjectListFullSearchParams] + required: + - _meta + - _links + - data + type: object + properties: + _meta: + $ref: '#/components/schemas/PageMetaInfoLimitOffset' + _links: + $ref: '#/components/schemas/PageLinks' + data: + title: Data + type: array + items: + $ref: '#/components/schemas/ProjectListFullSearchParams' + additionalProperties: false + description: Paginated response model of ItemTs Page_ProjectListItem_: title: Page[ProjectListItem] required: @@ -10414,6 +10451,37 @@ components: format: uri results: $ref: '#/components/schemas/ExtractedResults' + ProjectListFullSearchParams: + title: ProjectListFullSearchParams + type: object + properties: + limit: + title: Limit + exclusiveMaximum: true + minimum: 1 + type: integer + description: maximum number of items to return (pagination) + default: 20 + maximum: 50 + offset: + title: Offset + minimum: 0 + type: integer + description: index to the first item to return (pagination) + default: 0 + text: + title: Text + maxLength: 100 + type: string + description: Multi column full text search, across all folders and workspaces + example: My Project + tag_ids: + title: Tag Ids + type: string + description: Search by tag ID (multiple tag IDs may be provided separated + by column) + example: 1,3 + description: Use as pagination options in query parameters ProjectListItem: title: ProjectListItem required: diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index b3f2f928329..21802e9841d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -143,25 +143,54 @@ async def list_projects( # pylint: disable=too-many-arguments async def list_projects_full_search( - app, + request, *, user_id: UserID, product_name: str, offset: NonNegativeInt, limit: int, text: str | None, + order_by: OrderBy, + tag_ids_list: list[int], ) -> tuple[list[ProjectDict], int]: - db = ProjectDBAPI.get_from_app_context(app) + db = ProjectDBAPI.get_from_app_context(request.app) + + user_available_services: list[dict] = await get_services_for_user_in_product( + request.app, user_id, product_name, only_key_versions=True + ) - total_number_projects, db_projects = await db.list_projects_full_search( + ( + db_projects, + db_project_types, + total_number_projects, + ) = await db.list_projects_full_search( user_id=user_id, product_name=product_name, + filter_by_services=user_available_services, text=text, offset=offset, limit=limit, + order_by=order_by, + tag_ids_list=tag_ids_list, ) - return db_projects, total_number_projects + projects: list[ProjectDict] = await logged_gather( + *( + _append_fields( + request, + user_id=user_id, + project=prj, + is_template=prj_type == ProjectTypeDB.TEMPLATE, + workspace_access_rights=None, + model_schema_cls=ProjectListItem, + ) + for prj, prj_type in zip(db_projects, db_project_types) + ), + reraise=True, + max_concurrency=100, + ) + + return projects, total_number_projects async def get_project( diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index b56a1764f7d..d2cce731d21 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -58,7 +58,7 @@ ProjectActiveParams, ProjectCreateHeaders, ProjectCreateParams, - ProjectListFullSearchParams, + ProjectListFullSearchWithJsonStrParams, ProjectListWithJsonStrParams, ) from ._permalink_api import update_or_pop_permalink_in_project @@ -69,6 +69,7 @@ ProjectInvalidUsageError, ProjectNotFoundError, ProjectOwnerNotFoundInTheProjectAccessRightsError, + WrongTagIdsInQueryError, ) from .lock import get_project_locked_state from .models import ProjectDict @@ -101,7 +102,10 @@ async def _wrapper(request: web.Request) -> web.StreamResponse: WorkspaceNotFoundError, ) as exc: raise web.HTTPNotFound(reason=f"{exc}") from exc - except ProjectOwnerNotFoundInTheProjectAccessRightsError as exc: + except ( + ProjectOwnerNotFoundInTheProjectAccessRightsError, + WrongTagIdsInQueryError, + ) as exc: raise web.HTTPBadRequest(reason=f"{exc}") from exc except ( ProjectInvalidRightsError, @@ -233,17 +237,22 @@ async def list_projects(request: web.Request): @_handle_projects_exceptions async def list_projects_full_search(request: web.Request): req_ctx = RequestContext.parse_obj(request) - query_params: ProjectListFullSearchParams = parse_request_query_parameters_as( - ProjectListFullSearchParams, request + query_params: ProjectListFullSearchWithJsonStrParams = ( + parse_request_query_parameters_as( + ProjectListFullSearchWithJsonStrParams, request + ) ) + tag_ids_list = query_params.tag_ids_list() projects, total_number_of_projects = await _crud_api_read.list_projects_full_search( - request.app, + request, user_id=req_ctx.user_id, product_name=req_ctx.product_name, limit=query_params.limit, offset=query_params.offset, text=query_params.text, + order_by=query_params.order_by, + tag_ids_list=tag_ids_list, ) page = Page[ProjectDict].parse_obj( diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py index 967ca8fe090..2fdef2fb3e2 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py @@ -17,7 +17,15 @@ null_or_none_str_to_none_validator, ) from models_library.workspaces import WorkspaceID -from pydantic import BaseModel, Extra, Field, Json, root_validator, validator +from pydantic import ( + BaseModel, + Extra, + Field, + Json, + parse_obj_as, + root_validator, + validator, +) from servicelib.common_headers import ( UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, X_SIMCORE_PARENT_NODE_ID, @@ -25,6 +33,7 @@ X_SIMCORE_USER_AGENT, ) +from .exceptions import WrongTagIdsInQueryError from .models import ProjectTypeAPI @@ -123,7 +132,7 @@ def search_check_empty_string(cls, v): )(null_or_none_str_to_none_validator) -class ProjectListWithJsonStrParams(ProjectListParams): +class ProjectListWithOrderByParams(BaseModel): order_by: Json[OrderBy] = Field( # pylint: disable=unsubscriptable-object default=OrderBy(field=IDStr("last_change_date"), direction=OrderDirection.DESC), description="Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date) and direction (asc|desc). The default sorting order is ascending.", @@ -151,6 +160,10 @@ class Config: extra = Extra.forbid +class ProjectListWithJsonStrParams(ProjectListParams, ProjectListWithOrderByParams): + ... + + class ProjectActiveParams(BaseModel): client_session_id: str @@ -162,7 +175,30 @@ class ProjectListFullSearchParams(PageQueryParameters): max_length=100, example="My Project", ) + tag_ids: str | None = Field( + default=None, + description="Search by tag ID (multiple tag IDs may be provided separated by column)", + example="1,3", + ) _empty_is_none = validator("text", allow_reuse=True, pre=True)( empty_str_to_none_pre_validator ) + + +class ProjectListFullSearchWithJsonStrParams( + ProjectListFullSearchParams, ProjectListWithOrderByParams +): + def tag_ids_list(self) -> list[int]: + try: + # Split the tag_ids by commas and map them to integers + if self.tag_ids: + tag_ids_list = list(map(int, self.tag_ids.split(","))) + # Validate that the tag_ids_list is indeed a list of integers + parse_obj_as(list[int], tag_ids_list) + else: + tag_ids_list = [] + except ValueError as exc: + raise WrongTagIdsInQueryError from exc + + return tag_ids_list diff --git a/services/web/server/src/simcore_service_webserver/projects/_db_utils.py b/services/web/server/src/simcore_service_webserver/projects/_db_utils.py index aa0f6ce8048..8bda162ab6f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_db_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_db_utils.py @@ -20,7 +20,7 @@ from simcore_postgres_database.webserver_models import ProjectType, projects from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.sql import select -from sqlalchemy.sql.selectable import Select +from sqlalchemy.sql.selectable import CompoundSelect, Select from ..db.models import GroupType, groups, projects_tags, user_to_groups, users from ..users.exceptions import UserNotFoundError @@ -181,7 +181,7 @@ async def _execute_without_permission_check( conn: SAConnection, user_id: UserID, *, - select_projects_query: Select, + select_projects_query: Select | CompoundSelect, filter_by_services: list[dict] | None = None, ) -> tuple[list[dict[str, Any]], list[ProjectType]]: api_projects: list[dict] = [] # API model-compatible projects diff --git a/services/web/server/src/simcore_service_webserver/projects/db.py b/services/web/server/src/simcore_service_webserver/projects/db.py index 0e8b8c242fa..10487a80e90 100644 --- a/services/web/server/src/simcore_service_webserver/projects/db.py +++ b/services/web/server/src/simcore_service_webserver/projects/db.py @@ -40,6 +40,7 @@ from simcore_postgres_database.models.groups import user_to_groups from simcore_postgres_database.models.project_to_groups import project_to_groups from simcore_postgres_database.models.projects_nodes import projects_nodes +from simcore_postgres_database.models.projects_tags import projects_tags from simcore_postgres_database.models.projects_to_folders import projects_to_folders from simcore_postgres_database.models.projects_to_products import projects_to_products from simcore_postgres_database.models.wallets import wallets @@ -494,10 +495,15 @@ async def list_projects_full_search( *, user_id: PositiveInt, product_name: ProductName, + filter_by_services: list[dict] | None = None, text: str | None = None, offset: int | None = 0, limit: int | None = None, - ) -> tuple[int, list[ProjectDict]]: + tag_ids_list: list[int], + order_by: OrderBy = OrderBy( + field=IDStr("last_change_date"), direction=OrderDirection.DESC + ), + ) -> tuple[list[dict[str, Any]], list[ProjectType], int]: async with self.engine.acquire() as conn: user_groups: list[RowProxy] = await self._list_user_groups(conn, user_id) @@ -520,6 +526,13 @@ async def list_projects_full_search( ).group_by(workspaces_access_rights.c.workspace_id) ).subquery("workspace_access_rights_subquery") + project_tags_subquery = ( + sa.select( + projects_tags.c.project_id, + sa.func.array_agg(projects_tags.c.tag_id).label("tags"), + ).group_by(projects_tags.c.project_id) + ).subquery("project_tags_subquery") + private_workspace_query = ( sa.select( *[ @@ -530,6 +543,10 @@ async def list_projects_full_search( self.access_rights_subquery.c.access_rights, projects_to_products.c.product_name, projects_to_folders.c.folder_id, + sa.func.coalesce( + project_tags_subquery.c.tags, + sa.cast(sa.text("'{}'"), sa.ARRAY(sa.Integer)), + ).label("tags"), ) .select_from( projects.join(self.access_rights_subquery, isouter=True) @@ -542,6 +559,7 @@ async def list_projects_full_search( ), isouter=True, ) + .join(project_tags_subquery, isouter=True) ) .where( ( @@ -562,6 +580,14 @@ async def list_projects_full_search( ) ) + if tag_ids_list: + private_workspace_query = private_workspace_query.where( + sa.func.coalesce( + project_tags_subquery.c.tags, + sa.cast(sa.text("'{}'"), sa.ARRAY(sa.Integer)), + ).op("@>")(tag_ids_list) + ) + shared_workspace_query = ( sa.select( *[ @@ -572,6 +598,10 @@ async def list_projects_full_search( workspace_access_rights_subquery.c.access_rights, projects_to_products.c.product_name, projects_to_folders.c.folder_id, + sa.func.coalesce( + project_tags_subquery.c.tags, + sa.cast(sa.text("'{}'"), sa.ARRAY(sa.Integer)), + ).label("tags"), ) .select_from( projects.join( @@ -588,6 +618,7 @@ async def list_projects_full_search( ), isouter=True, ) + .join(project_tags_subquery, isouter=True) ) .where( ( @@ -607,6 +638,14 @@ async def list_projects_full_search( ) ) + if tag_ids_list: + shared_workspace_query = shared_workspace_query.where( + sa.func.coalesce( + project_tags_subquery.c.tags, + sa.cast(sa.text("'{}'"), sa.ARRAY(sa.Integer)), + ).op("@>")(tag_ids_list) + ) + combined_query = sa.union_all( private_workspace_query, shared_workspace_query ) @@ -614,23 +653,27 @@ async def list_projects_full_search( count_query = sa.select(func.count()).select_from(combined_query) total_count = await conn.scalar(count_query) - list_query = combined_query.offset(offset).limit(limit) - result = await conn.execute(list_query) - rows = await result.fetchall() or [] - results: list[UserSpecificProjectDataDB] = [ - UserSpecificProjectDataDB.from_orm(row) for row in rows - ] + if order_by.direction == OrderDirection.ASC: + combined_query = combined_query.order_by( + sa.asc(getattr(projects.c, order_by.field)) + ) + else: + combined_query = combined_query.order_by( + sa.desc(getattr(projects.c, order_by.field)) + ) - # NOTE: Additional adjustments to make it back compatible - list_of_converted_projects: list[ProjectDict] = [] - for project in results: - user_email = await self._get_user_email(conn, project.prj_owner) - converted_project = convert_to_schema_names(project.dict(), user_email) - converted_project["tags"] = [] # <-- Tags not needed for now - converted_project["state"] = None - list_of_converted_projects.append(converted_project) + prjs, prj_types = await self._execute_without_permission_check( + conn, + user_id=user_id, + select_projects_query=combined_query.offset(offset).limit(limit), + filter_by_services=filter_by_services, + ) - return cast(int, total_count), list_of_converted_projects + return ( + prjs, + prj_types, + cast(int, total_count), + ) async def list_projects_uuids(self, user_id: int) -> list[str]: async with self.engine.acquire() as conn: diff --git a/services/web/server/src/simcore_service_webserver/projects/exceptions.py b/services/web/server/src/simcore_service_webserver/projects/exceptions.py index 7b8ff61a971..871628ee66d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/exceptions.py +++ b/services/web/server/src/simcore_service_webserver/projects/exceptions.py @@ -30,6 +30,10 @@ class ProjectOwnerNotFoundInTheProjectAccessRightsError(BaseProjectError): msg_template = "Project owner gid with required permissions was not found in the project access rights" +class WrongTagIdsInQueryError(BaseProjectError): + msg_template = "Wrong value in `tag_ids` query parameter" + + class ProjectInvalidRightsError(BaseProjectError): msg_template = ( "User '{user_id}' has no rights to access project with uuid '{project_uuid}'" diff --git a/services/web/server/tests/unit/with_dbs/03/workspaces/test_workspaces__list_projects_full_search.py b/services/web/server/tests/unit/with_dbs/03/workspaces/test_workspaces__list_projects_full_search.py index 1340630e489..a7efd64b485 100644 --- a/services/web/server/tests/unit/with_dbs/03/workspaces/test_workspaces__list_projects_full_search.py +++ b/services/web/server/tests/unit/with_dbs/03/workspaces/test_workspaces__list_projects_full_search.py @@ -5,6 +5,7 @@ # pylint: disable=too-many-statements +import json from copy import deepcopy from http import HTTPStatus @@ -87,6 +88,8 @@ async def test_workspaces__list_projects_full_search( assert data[0]["uuid"] == project_1["uuid"] assert data[0]["workspaceId"] == added_workspace["workspaceId"] assert data[0]["folderId"] is None + assert data[0]["workbench"] + assert data[0]["accessRights"] # Create projects in private workspace project_data = deepcopy(fake_project) @@ -160,3 +163,76 @@ async def test_workspaces__list_projects_full_search( assert sorted_data[2]["uuid"] == project_3["uuid"] assert sorted_data[2]["workspaceId"] is None assert sorted_data[2]["folderId"] == root_folder["folderId"] + + +@pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) +async def test__list_projects_full_search_with_query_parameters( + client: TestClient, + logged_user: UserInfoDict, + user_project: ProjectDict, + expected: HTTPStatus, + mock_catalog_api_get_services_for_user_in_product: MockerFixture, + fake_project: ProjectDict, + workspaces_clean_db: None, +): + assert client.app + + # Create projects in private workspace + project_data = deepcopy(fake_project) + project_data["name"] = _SEARCH_NAME_2 + project = await create_project( + client.app, + project_data, + user_id=logged_user["id"], + product_name="osparc", + ) + + # Full search with text + base_url = client.app.router["list_projects_full_search"].url_for() + url = base_url.with_query({"text": "Orion"}) + resp = await client.get(url) + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + assert data[0]["uuid"] == project["uuid"] + + # Full search with order_by + base_url = client.app.router["list_projects_full_search"].url_for() + url = base_url.with_query( + { + "text": "Orion", + "order_by": json.dumps({"field": "uuid", "direction": "desc"}), + } + ) + resp = await client.get(url) + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + assert data[0]["uuid"] == project["uuid"] + + # Full search with tag_ids + base_url = client.app.router["list_projects_full_search"].url_for() + url = base_url.with_query({"text": "Orion", "tag_ids": "1,2"}) + resp = await client.get(url) + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 0 + + # Create tag + url = client.app.router["create_tag"].url_for() + resp = await client.post( + f"{url}", json={"name": "tag1", "description": "description1", "color": "#f00"} + ) + added_tag, _ = await assert_status(resp, expected) + + # Add tag to study + url = client.app.router["add_project_tag"].url_for( + project_uuid=project["uuid"], tag_id=str(added_tag.get("id")) + ) + resp = await client.post(f"{url}") + data, _ = await assert_status(resp, expected) + + # Full search with tag_ids + base_url = client.app.router["list_projects_full_search"].url_for() + url = base_url.with_query({"text": "Orion", "tag_ids": f"{added_tag['id']}"}) + resp = await client.get(url) + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + assert data[0]["uuid"] == project["uuid"]