From 641ef5170ec7625c241f396019c4026875057b93 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 5 Dec 2024 17:52:39 -0800 Subject: [PATCH 1/2] Escape datetime and ObjectId values in test privacy results --- .../v1/endpoints/privacy_request_endpoints.py | 9 +- .../test_privacy_request_endpoints.py | 206 +++++++++++++++++- 2 files changed, 210 insertions(+), 5 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index c8dc17a273..84cb453a63 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -2,6 +2,7 @@ import csv import io +import json from collections import defaultdict from datetime import datetime from typing import ( @@ -148,6 +149,7 @@ from fides.api.util.enums import ColumnSort from fides.api.util.fuzzy_search_utils import get_decrypted_identities_automaton from fides.api.util.logger import Pii +from fides.api.util.storage_util import storage_json_encoder from fides.common.api.scope_registry import ( PRIVACY_REQUEST_CALLBACK_RESUME, PRIVACY_REQUEST_CREATE, @@ -2657,8 +2659,13 @@ def get_test_privacy_request_results( ) privacy_request.save(db=db) + # Escape datetime and ObjectId values + raw_data = privacy_request.get_raw_access_results() + escaped_json = json.dumps(raw_data, indent=2, default=storage_json_encoder) + escaped_data = json.loads(escaped_json) + return { "privacy_request_id": privacy_request.id, "status": privacy_request.status, - "results": privacy_request.get_raw_access_results(), + "results": escaped_data, } diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 21007687ec..8ab833ec08 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -8365,7 +8365,6 @@ def test_get_access_results_contributor_but_disabled( @pytest.mark.integration -@pytest.mark.integration_postgres class TestPrivacyRequestFilteredResults: @pytest.fixture(scope="function") def default_access_policy(self, db) -> None: @@ -8427,15 +8426,18 @@ def test_filtered_results_with_roles( ) assert response.status_code == expected_status + @pytest.mark.integration_postgres @pytest.mark.usefixtures("default_access_policy", "postgres_integration_db") - def test_filtered_results( + def test_filtered_results_postgres( self, connection_config, - dataset_config, + postgres_example_test_dataset_config, api_client: TestClient, generate_auth_header, ) -> None: - dataset_url = get_connection_dataset_url(connection_config, dataset_config) + dataset_url = get_connection_dataset_url( + connection_config, postgres_example_test_dataset_config + ) auth_header = generate_auth_header(scopes=[DATASET_TEST]) response = api_client.post( dataset_url + "/test", @@ -8459,3 +8461,199 @@ def test_filtered_results( "status", "results", } + assert response.json()["results"] == { + "postgres_example_test_dataset:customer": [ + { + "address_id": 4, + "created": "2020-04-01T11:47:42", + "email": "jane@example.com", + "id": 3, + "name": "Jane Customer", + } + ], + "postgres_example_test_dataset:employee": [], + "postgres_example_test_dataset:report": [], + "postgres_example_test_dataset:visit": [], + "postgres_example_test_dataset:login": [ + {"customer_id": 3, "id": 8, "time": "2021-01-06T01:00:00"} + ], + "postgres_example_test_dataset:orders": [ + {"customer_id": 3, "id": "ord_ddd-eee", "shipping_address_id": 4} + ], + "postgres_example_test_dataset:payment_card": [ + { + "billing_address_id": 4, + "ccn": 373719391, + "code": 222, + "customer_id": 3, + "id": "pay_ccc-ccc", + "name": "Example Card 3", + "preferred": False, + } + ], + "postgres_example_test_dataset:service_request": [], + "postgres_example_test_dataset:order_item": [], + "postgres_example_test_dataset:address": [ + { + "city": "Example Mountain", + "house": 1111, + "id": 4, + "state": "TX", + "street": "Example Place", + "zip": "54321", + } + ], + "postgres_example_test_dataset:product": [], + } + + @pytest.mark.integration_mongo + @pytest.mark.usefixtures("default_access_policy") + def test_filtered_results_mongo( + self, + mongo_connection_config, + mongo_dataset_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + dataset_url = get_connection_dataset_url( + mongo_connection_config, mongo_dataset_config + ) + auth_header = generate_auth_header(scopes=[DATASET_TEST]) + response = api_client.post( + dataset_url + "/test", + headers=auth_header, + json={ + "email": "employee-1@example.com", + "postgres_example_test_dataset:customer:id": 1, + }, + ) + assert response.status_code == HTTP_200_OK + + privacy_request_id = response.json()["privacy_request_id"] + url = V1_URL_PREFIX + PRIVACY_REQUEST_FILTERED_RESULTS.format( + privacy_request_id=privacy_request_id + ) + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ_ACCESS_RESULTS]) + response = api_client.get( + url, + headers=auth_header, + ) + assert response.status_code == HTTP_200_OK + assert set(response.json().keys()) == { + "privacy_request_id", + "status", + "results", + } + assert response.json()["results"] == { + "mongo_test:customer_details": [ + { + "_id": {"$oid": "67525147dadb505a0d1751c6"}, + "customer_id": 1.0, + "customer_uuid": "3b241101-e2bb-4255-8caf-4136c566a962", + "gender": "male", + "birthday": "1988-01-10T00:00:00", + "workplace_info": { + "employer": "Mountain Baking Company", + "position": "Chief Strategist", + "direct_reports": ["Robbie Margo", "Sully Hunter"], + }, + "emergency_contacts": [ + { + "name": "June Customer", + "relationship": "mother", + "phone": "444-444-4444", + }, + { + "name": "Josh Customer", + "relationship": "brother", + "phone": "111-111-111", + }, + ], + "children": ["Christopher Customer", "Courtney Customer"], + "travel_identifiers": ["A111-11111", "B111-11111"], + "comments": [ + {"comment_id": "com_0001"}, + {"comment_id": "com_0003"}, + {"comment_id": "com_0005"}, + ], + } + ], + "mongo_test:customer_feedback": [], + "mongo_test:conversations": [ + { + "_id": {"$oid": "67525147dadb505a0d1751ce"}, + "thread": [ + { + "comment": "com_0001", + "message": "hello, testing in-flight chat feature", + "chat_name": "John C", + "ccn": "123456789", + } + ], + }, + { + "_id": {"$oid": "67525147dadb505a0d1751cf"}, + "thread": [ + { + "comment": "com_0003", + "message": "can I borrow your headphones?", + "chat_name": "John C", + "ccn": "123456789", + }, + { + "comment": "com_0005", + "message": "did you bring anything to read?", + "chat_name": "John C", + "ccn": "123456789", + }, + ], + }, + ], + "mongo_test:flights": [ + { + "_id": {"$oid": "67525147dadb505a0d1751d1"}, + "passenger_information": { + "passenger_ids": ["A111-11111"], + "full_name": "John Customer", + }, + "flight_no": "AA230", + "date": "2021-01-01", + "pilots": ["1", "2"], + "plane": 10002, + } + ], + "mongo_test:internal_customer_profile": [], + "mongo_test:payment_card": [ + { + "_id": {"$oid": "67525147dadb505a0d1751dd"}, + "id": "pay_aaa-aaa", + "name": "Example Card 1", + "ccn": "123456789", + "code": "321", + "preferred": True, + "customer_id": 1, + } + ], + "mongo_test:employee": [ + { + "_id": {"$oid": "67525147dadb505a0d1751d5"}, + "email": "employee-1@example.com", + "name": "Jack Employee", + "id": "1", + }, + { + "_id": {"$oid": "67525147dadb505a0d1751d6"}, + "email": "employee-2@example.com", + "name": "Jane Employee", + "id": "2", + }, + ], + "mongo_test:aircraft": [ + { + "_id": {"$oid": "67525147dadb505a0d1751d3"}, + "model": "Airbus A350", + "planes": ["10002"], + } + ], + "mongo_test:rewards": [], + } From ffe64d392b6d921882ae4bd76c6355db6fc2b4d6 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Fri, 6 Dec 2024 08:07:01 -0800 Subject: [PATCH 2/2] Removing dynamic payloads from assertion --- .../test_privacy_request_endpoints.py | 157 ------------------ 1 file changed, 157 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 8ab833ec08..aaf64d4e7b 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -8461,50 +8461,6 @@ def test_filtered_results_postgres( "status", "results", } - assert response.json()["results"] == { - "postgres_example_test_dataset:customer": [ - { - "address_id": 4, - "created": "2020-04-01T11:47:42", - "email": "jane@example.com", - "id": 3, - "name": "Jane Customer", - } - ], - "postgres_example_test_dataset:employee": [], - "postgres_example_test_dataset:report": [], - "postgres_example_test_dataset:visit": [], - "postgres_example_test_dataset:login": [ - {"customer_id": 3, "id": 8, "time": "2021-01-06T01:00:00"} - ], - "postgres_example_test_dataset:orders": [ - {"customer_id": 3, "id": "ord_ddd-eee", "shipping_address_id": 4} - ], - "postgres_example_test_dataset:payment_card": [ - { - "billing_address_id": 4, - "ccn": 373719391, - "code": 222, - "customer_id": 3, - "id": "pay_ccc-ccc", - "name": "Example Card 3", - "preferred": False, - } - ], - "postgres_example_test_dataset:service_request": [], - "postgres_example_test_dataset:order_item": [], - "postgres_example_test_dataset:address": [ - { - "city": "Example Mountain", - "house": 1111, - "id": 4, - "state": "TX", - "street": "Example Place", - "zip": "54321", - } - ], - "postgres_example_test_dataset:product": [], - } @pytest.mark.integration_mongo @pytest.mark.usefixtures("default_access_policy") @@ -8544,116 +8500,3 @@ def test_filtered_results_mongo( "status", "results", } - assert response.json()["results"] == { - "mongo_test:customer_details": [ - { - "_id": {"$oid": "67525147dadb505a0d1751c6"}, - "customer_id": 1.0, - "customer_uuid": "3b241101-e2bb-4255-8caf-4136c566a962", - "gender": "male", - "birthday": "1988-01-10T00:00:00", - "workplace_info": { - "employer": "Mountain Baking Company", - "position": "Chief Strategist", - "direct_reports": ["Robbie Margo", "Sully Hunter"], - }, - "emergency_contacts": [ - { - "name": "June Customer", - "relationship": "mother", - "phone": "444-444-4444", - }, - { - "name": "Josh Customer", - "relationship": "brother", - "phone": "111-111-111", - }, - ], - "children": ["Christopher Customer", "Courtney Customer"], - "travel_identifiers": ["A111-11111", "B111-11111"], - "comments": [ - {"comment_id": "com_0001"}, - {"comment_id": "com_0003"}, - {"comment_id": "com_0005"}, - ], - } - ], - "mongo_test:customer_feedback": [], - "mongo_test:conversations": [ - { - "_id": {"$oid": "67525147dadb505a0d1751ce"}, - "thread": [ - { - "comment": "com_0001", - "message": "hello, testing in-flight chat feature", - "chat_name": "John C", - "ccn": "123456789", - } - ], - }, - { - "_id": {"$oid": "67525147dadb505a0d1751cf"}, - "thread": [ - { - "comment": "com_0003", - "message": "can I borrow your headphones?", - "chat_name": "John C", - "ccn": "123456789", - }, - { - "comment": "com_0005", - "message": "did you bring anything to read?", - "chat_name": "John C", - "ccn": "123456789", - }, - ], - }, - ], - "mongo_test:flights": [ - { - "_id": {"$oid": "67525147dadb505a0d1751d1"}, - "passenger_information": { - "passenger_ids": ["A111-11111"], - "full_name": "John Customer", - }, - "flight_no": "AA230", - "date": "2021-01-01", - "pilots": ["1", "2"], - "plane": 10002, - } - ], - "mongo_test:internal_customer_profile": [], - "mongo_test:payment_card": [ - { - "_id": {"$oid": "67525147dadb505a0d1751dd"}, - "id": "pay_aaa-aaa", - "name": "Example Card 1", - "ccn": "123456789", - "code": "321", - "preferred": True, - "customer_id": 1, - } - ], - "mongo_test:employee": [ - { - "_id": {"$oid": "67525147dadb505a0d1751d5"}, - "email": "employee-1@example.com", - "name": "Jack Employee", - "id": "1", - }, - { - "_id": {"$oid": "67525147dadb505a0d1751d6"}, - "email": "employee-2@example.com", - "name": "Jane Employee", - "id": "2", - }, - ], - "mongo_test:aircraft": [ - { - "_id": {"$oid": "67525147dadb505a0d1751d3"}, - "model": "Airbus A350", - "planes": ["10002"], - } - ], - "mongo_test:rewards": [], - }