diff --git a/CHANGELOG.md b/CHANGELOG.md index d205a4d4a0..919c1d95b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The types of changes are: - Adding hashes to system tab URLs [#5535](https://github.com/ethyca/fides/pull/5535) - Boolean inputs will now show as a select with true/false values in the connection form [#5555](https://github.com/ethyca/fides/pull/5555) - Updated Cookie House to be responsive [#5541](https://github.com/ethyca/fides/pull/5541) +- Updated `/system` endpoint to filter vendor deleted systems [#5553](https://github.com/ethyca/fides/pull/5553) ### Developer Experience - Migrated remaining instances of Chakra's Select component to use Ant's Select component [#5502](https://github.com/ethyca/fides/pull/5502) diff --git a/src/fides/api/api/v1/endpoints/system.py b/src/fides/api/api/v1/endpoints/system.py index 93093c63d7..0171265eda 100644 --- a/src/fides/api/api/v1/endpoints/system.py +++ b/src/fides/api/api/v1/endpoints/system.py @@ -1,3 +1,4 @@ +import datetime from typing import Annotated, Dict, List, Optional, Union from fastapi import Depends, HTTPException, Query, Response, Security @@ -9,6 +10,7 @@ from fideslang.validation import FidesKey from loguru import logger from pydantic import Field +from sqlalchemy import or_ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.orm import Session @@ -17,11 +19,7 @@ from fides.api.api import deps from fides.api.api.v1.endpoints.saas_config_endpoints import instantiate_connection -from fides.api.db.crud import ( - get_resource, - get_resource_with_custom_fields, - list_resource, -) +from fides.api.db.crud import get_resource, get_resource_with_custom_fields from fides.api.db.ctl_session import get_async_db from fides.api.db.system import ( create_system, @@ -391,30 +389,23 @@ async def ls( # pylint: disable=invalid-name data_subjects: Optional[List[FidesKey]] = Query(None), dnd_relevant: Optional[bool] = Query(None), show_hidden: Optional[bool] = Query(False), + show_deleted: Optional[bool] = Query(False), ) -> List: """Get a list of all of the Systems. If any parameters or filters are provided the response will be paginated and/or filtered. Otherwise all Systems will be returned (this may be a slow operation if there are many systems, so using the pagination parameters is recommended). """ - if not ( - size - or page - or search - or data_uses - or data_categories - or data_subjects - or dnd_relevant - or show_hidden - ): - return await list_resource(System, db) + + query = select(System) pagination_params = Params(page=page or 1, size=size or 50) # Need to join with PrivacyDeclaration in order to be able to filter # by data use, data category, and data subject - query = select(System).outerjoin( - PrivacyDeclaration, System.id == PrivacyDeclaration.system_id - ) + if any([data_uses, data_categories, data_subjects]): + query = query.outerjoin( + PrivacyDeclaration, System.id == PrivacyDeclaration.system_id + ) # Fetch any system that is relevant for Detection and Discovery, ie any of the following: # - has connection configurations (has some integration for DnD or SaaS) @@ -431,6 +422,15 @@ async def ls( # pylint: disable=invalid-name System.hidden == False # pylint: disable=singleton-comparison ) + # Filter out any vendor deleted systems, unless explicitly asked for + if not show_deleted: + query = query.filter( + or_( + System.vendor_deleted_date.is_(None), + System.vendor_deleted_date >= datetime.datetime.now(), + ) + ) + filter_params = FilterParams( search=search, data_uses=data_uses, @@ -446,6 +446,20 @@ async def ls( # pylint: disable=invalid-name # Add a distinct so we only get one row per system duplicates_removed = filtered_query.distinct(System.id) + + if not ( + size + or page + or search + or data_uses + or data_categories + or data_subjects + or dnd_relevant + or show_hidden + ): + result = await db.execute(duplicates_removed) + return result.scalars().all() + return await async_paginate(db, duplicates_removed, pagination_params) diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index 0cdb9e2419..bfca14dd84 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -2,7 +2,7 @@ """Integration tests for the API module.""" import json import typing -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from json import loads from typing import Dict, List, Tuple from uuid import uuid4 @@ -1665,6 +1665,40 @@ def test_list_with_pagination_and_multiple_filters_2( assert result_json["items"][0]["fides_key"] == tcf_system.fides_key + @pytest.mark.parametrize( + "vendor_deleted_date, expected_systems_count, show_deleted", + [ + (datetime.now() - timedelta(days=1), 1, True), + (datetime.now() - timedelta(days=1), 0, None), + (datetime.now() + timedelta(days=1), 1, None), + (None, 1, None), + ], + ) + def test_vendor_deleted_systems( + self, + db, + test_config, + system_with_cleanup, + vendor_deleted_date, + expected_systems_count, + show_deleted, + ): + + system_with_cleanup.vendor_deleted_date = vendor_deleted_date + db.commit() + + result = _api.ls( + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + resource_type="system", + query_params={"show_deleted": True} if show_deleted else {}, + ) + + assert result.status_code == 200 + result_json = result.json() + + assert len(result_json) == expected_systems_count + @pytest.mark.unit class TestSystemUpdate: