From 776cbcbf15bf682968f3013d3c2196db336bc606 Mon Sep 17 00:00:00 2001 From: Jason Gill Date: Thu, 12 Dec 2024 08:24:24 -0700 Subject: [PATCH 1/3] Set up Flutter listeners and callHandlers in the Sample App for native app communication (#5597) --- CHANGELOG.md | 1 + .../src/components/notices/NoticeOverlay.tsx | 4 ++-- .../src/components/tcf/TcfOverlay.tsx | 8 +++---- clients/sample-app/package-lock.json | 2 +- clients/sample-app/package.json | 2 +- .../sample-app/src/pages/embedded-consent.tsx | 24 ++++++++++++++----- clients/sample-app/src/pages/index.tsx | 9 +++++++ 7 files changed, 36 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24f0e34ecb..ad2999fea3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The types of changes are: ### Added - New page in the Cookie House sample app to demonstrate the use of embedding the FidesJS SDK on the page [#5564](https://github.com/ethyca/fides/pull/5564) +- Added event based communication example to the Cookie House sample app [#5597](https://github.com/ethyca/fides/pull/5597) - Added new erasure tests for BigQuery Enterprise [#5554](https://github.com/ethyca/fides/pull/5554) ### Fixed diff --git a/clients/fides-js/src/components/notices/NoticeOverlay.tsx b/clients/fides-js/src/components/notices/NoticeOverlay.tsx index b24ca220b3..44a31caf7a 100644 --- a/clients/fides-js/src/components/notices/NoticeOverlay.tsx +++ b/clients/fides-js/src/components/notices/NoticeOverlay.tsx @@ -258,12 +258,12 @@ const NoticeOverlay: FunctionComponent = ({ if (isConsentOverride(options) && experience.privacy_notices) { if (options.fidesConsentOverride === ConsentMethod.ACCEPT) { fidesDebugger( - "Consent automatically accepted by fides_accept_all override!", + "Consent automatically accepted by fides_consent_override!", ); handleAcceptAll(true); } else if (options.fidesConsentOverride === ConsentMethod.REJECT) { fidesDebugger( - "Consent automatically rejected by fides_reject_all override!", + "Consent automatically rejected by fides_consent_override!", ); handleRejectAll(true); } diff --git a/clients/fides-js/src/components/tcf/TcfOverlay.tsx b/clients/fides-js/src/components/tcf/TcfOverlay.tsx index 907fc613b6..9e47bf35f7 100644 --- a/clients/fides-js/src/components/tcf/TcfOverlay.tsx +++ b/clients/fides-js/src/components/tcf/TcfOverlay.tsx @@ -362,12 +362,12 @@ export const TcfOverlay = ({ useEffect(() => { if (options.fidesConsentOverride === ConsentMethod.ACCEPT) { fidesDebugger( - "Consent automatically accepted by fides_accept_all override!", + "Consent automatically accepted by fides_consent_override!", ); handleAcceptAll(true); } else if (options.fidesConsentOverride === ConsentMethod.REJECT) { fidesDebugger( - "Consent automatically rejected by fides_reject_all override!", + "Consent automatically rejected by fides_consent_override!", ); handleRejectAll(true); } @@ -389,7 +389,7 @@ export const TcfOverlay = ({ }, [cookie, options.debug]); const handleDismiss = useCallback(() => { - handleUpdateAllPreferences(ConsentMethod.DISMISS, draftIds!); + handleUpdateAllPreferences(ConsentMethod.DISMISS, draftIds); }, [handleUpdateAllPreferences, draftIds]); const experienceConfig = @@ -461,7 +461,7 @@ export const TcfOverlay = ({ : () => ( { setDraftIds(updatedIds); dispatchFidesEvent("FidesUIChanged", cookie, options.debug); diff --git a/clients/sample-app/package-lock.json b/clients/sample-app/package-lock.json index d513b3a1d4..7ed686f9c1 100644 --- a/clients/sample-app/package-lock.json +++ b/clients/sample-app/package-lock.json @@ -14,7 +14,7 @@ "react": "18.3.1", "react-dom": "18.3.1", "react-hook-form": "7.38.0", - "react-select": "^5.7.3", + "react-select": "^5.8.3", "sass": "1.55.0" }, "devDependencies": { diff --git a/clients/sample-app/package.json b/clients/sample-app/package.json index b4f8cabd47..d74ce250f2 100644 --- a/clients/sample-app/package.json +++ b/clients/sample-app/package.json @@ -22,7 +22,7 @@ "react": "18.3.1", "react-dom": "18.3.1", "react-hook-form": "7.38.0", - "react-select": "^5.7.3", + "react-select": "^5.8.3", "sass": "1.55.0" }, "devDependencies": { diff --git a/clients/sample-app/src/pages/embedded-consent.tsx b/clients/sample-app/src/pages/embedded-consent.tsx index 3b4334b031..9f5049ce5b 100644 --- a/clients/sample-app/src/pages/embedded-consent.tsx +++ b/clients/sample-app/src/pages/embedded-consent.tsx @@ -61,14 +61,17 @@ const IndexPage = ({ gtmContainerId, privacyCenterUrl }: Props) => { {/* Allow the embedded consent modal to fill the screen */} @@ -101,6 +104,15 @@ const IndexPage = ({ gtmContainerId, privacyCenterUrl }: Props) => { `} ) : null} + {/* Support for Flutter InAppWebView communication https://inappwebview.dev/docs/webview/javascript/communication */} + {/* eslint-disable-next-line @next/next/no-before-interactive-script-outside-document */} +
); diff --git a/clients/sample-app/src/pages/index.tsx b/clients/sample-app/src/pages/index.tsx index 0e91ab7920..44a8107aa2 100644 --- a/clients/sample-app/src/pages/index.tsx +++ b/clients/sample-app/src/pages/index.tsx @@ -98,6 +98,15 @@ const IndexPage = ({ gtmContainerId, privacyCenterUrl, products }: Props) => { `} ) : null} + {/* Support for Flutter InAppWebView communication https://inappwebview.dev/docs/webview/javascript/communication */} + {/* eslint-disable-next-line @next/next/no-before-interactive-script-outside-document */} + ); From ad6a9c31afafae25426272a72e2ff335be018656 Mon Sep 17 00:00:00 2001 From: Andres Torres Date: Thu, 12 Dec 2024 09:39:48 -0600 Subject: [PATCH 2/3] HJ-294 - Fix Safe-Tests(ctl-not-external) on CI is timing out (#5568) --- src/fides/api/api/v1/endpoints/admin.py | 10 +++++++++- src/fides/api/app_setup.py | 9 ++++++--- src/fides/api/db/database.py | 1 + tests/ctl/cli/test_cli.py | 5 ++++- tests/ctl/conftest.py | 15 +++++++++++---- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/admin.py b/src/fides/api/api/v1/endpoints/admin.py index 0225d7a9d1..7027aa8f5d 100644 --- a/src/fides/api/api/v1/endpoints/admin.py +++ b/src/fides/api/api/v1/endpoints/admin.py @@ -66,7 +66,15 @@ async def db_action(action: DBActions, revision: Optional[str] = "head") -> Dict reset_db(CONFIG.database.sync_database_uri) action_text = "reset" - await configure_db(CONFIG.database.sync_database_uri, revision=revision) + try: + logger.info("Database being configured...") + await configure_db(CONFIG.database.sync_database_uri, revision=revision) + except Exception as e: + logger.exception("Database configuration failed: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Database configuration failed: {e}. Check server logs for more details", + ) return { "data": { diff --git a/src/fides/api/app_setup.py b/src/fides/api/app_setup.py index 17f49fdd19..fbab3819a7 100644 --- a/src/fides/api/app_setup.py +++ b/src/fides/api/app_setup.py @@ -163,9 +163,12 @@ async def run_database_startup(app: FastAPI) -> None: raise FidesError("No database uri provided") if CONFIG.database.automigrate: - await configure_db( - CONFIG.database.sync_database_uri, samples=CONFIG.database.load_samples - ) + try: + await configure_db( + CONFIG.database.sync_database_uri, samples=CONFIG.database.load_samples + ) + except Exception as e: + logger.error("Error occurred during database configuration: {}", str(e)) else: logger.info("Skipping auto-migration due to 'automigrate' configuration value.") diff --git a/src/fides/api/db/database.py b/src/fides/api/db/database.py index 229405f18a..8bf4d3e699 100644 --- a/src/fides/api/db/database.py +++ b/src/fides/api/db/database.py @@ -138,3 +138,4 @@ async def configure_db( error_type = get_full_exception_name(error) log.error("Unable to configure database: {}: {}", error_type, error) log.opt(exception=True).error(error) + raise diff --git a/tests/ctl/cli/test_cli.py b/tests/ctl/cli/test_cli.py index 2fbea40718..1d2e4938bb 100644 --- a/tests/ctl/cli/test_cli.py +++ b/tests/ctl/cli/test_cli.py @@ -145,13 +145,16 @@ def test_parse(test_config_path: str, test_cli_runner: CliRunner) -> None: class TestDB: + @pytest.mark.skip( + "This test is timing out only in CI: Safe-Tests (3.10.13, ctl-not-external)" + ) @pytest.mark.integration def test_reset_db(self, test_config_path: str, test_cli_runner: CliRunner) -> None: result = test_cli_runner.invoke( cli, ["-f", test_config_path, "db", "reset", "-y"] ) print(result.output) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output @pytest.mark.integration def test_init_db(self, test_config_path: str, test_cli_runner: CliRunner) -> None: diff --git a/tests/ctl/conftest.py b/tests/ctl/conftest.py index a4408feb45..db9f33e2ef 100644 --- a/tests/ctl/conftest.py +++ b/tests/ctl/conftest.py @@ -45,11 +45,18 @@ def monkeypatch_requests(test_client, monkeysession) -> None: @pytest.fixture(scope="session", autouse=True) -def setup_db(api_client, config): - """Apply migrations at beginning and end of testing session""" +@pytest.mark.usefixtures("monkeypatch_requests") +def setup_ctl_db(test_config, test_client, config): + "Sets up the database for testing." assert config.test_mode - assert requests.post != api_client.post - yield api_client.post(url=f"{config.cli.server_url}/v1/admin/db/reset") + assert ( + requests.post == test_client.post + ) # Sanity check to make sure monkeypatch_requests fixture has run + yield api.db_action( + server_url=test_config.cli.server_url, + headers=config.user.auth_header, + action="reset", + ) @pytest.fixture(scope="session") From 7e1832c802e2d3a71ab995e078156918b1b2306e Mon Sep 17 00:00:00 2001 From: Bruno Gutierrez Rios Date: Thu, 12 Dec 2024 13:19:19 -0300 Subject: [PATCH 3/3] La 202 update testing for flushing oauth tokens (#5587) --- .../schemas/saas/strategy_configuration.py | 13 +++- tests/fixtures/saas_example_fixtures.py | 70 ++++++++++++++++++ tests/ops/api/v1/endpoints/test_system.py | 66 ++++++++++++++--- ...tion_strategy_oauth2_client_credentials.py | 73 ------------------- 4 files changed, 136 insertions(+), 86 deletions(-) diff --git a/src/fides/api/schemas/saas/strategy_configuration.py b/src/fides/api/schemas/saas/strategy_configuration.py index 9bb5aaef17..368441dcb8 100644 --- a/src/fides/api/schemas/saas/strategy_configuration.py +++ b/src/fides/api/schemas/saas/strategy_configuration.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Union -from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from fides.api.schemas.saas.saas_config import Header, QueryParam, SaaSRequest from fides.api.schemas.saas.shared_schemas import ( @@ -154,8 +154,19 @@ class OAuth2BaseConfiguration(StrategyConfiguration): class OAuth2AuthorizationCodeConfiguration(OAuth2BaseConfiguration): """ + Oauth Authorization that requires manual user interaction to get authorization The standard OAuth2 configuration but with an additional property to configure the authorization request for the Authorization Code flow. """ authorization_request: SaaSRequest + + +class OAuth2ClientCredentialsConfiguration(OAuth2BaseConfiguration): + """ + Ouath authorization that does not require manual user interation to get authorization + The standard OAuth2 configuration, but excluding the refresh token during logging + since the client credentials flow does not require a refresh token. + """ + + refresh_request: Optional[SaaSRequest] = Field(exclude=True) diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index a973408372..3220adbd3d 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -19,6 +19,7 @@ from fides.api.schemas.saas.saas_config import ParamValue from fides.api.schemas.saas.strategy_configuration import ( OAuth2AuthorizationCodeConfiguration, + OAuth2ClientCredentialsConfiguration, ) from fides.api.service.masking.strategy.masking_strategy_nullify import ( NullMaskingStrategy, @@ -390,6 +391,75 @@ def oauth2_authorization_code_connection_config( connection_config.delete(db) +## TODO: base on the previous connection config to set up a new improved + + +@pytest.fixture(scope="function") +def oauth2_client_credentials_configuration() -> OAuth2ClientCredentialsConfiguration: + return { + "token_request": { + "method": "POST", + "path": "/oauth/token", + "headers": [ + { + "name": "Content-Type", + "value": "application/x-www-form-urlencoded", + } + ], + "query_params": [ + {"name": "client_id", "value": ""}, + {"name": "client_secret", "value": ""}, + {"name": "grant_type", "value": "client_credentials"}, + ], + }, + } + + +@pytest.fixture(scope="function") +def oauth2_client_credentials_connection_config( + db: Session, oauth2_client_credentials_configuration +) -> Generator: + secrets = { + "domain": "localhost", + "client_id": "client", + "client_secret": "secret", + "access_token": "access", + } + saas_config = { + "fides_key": "oauth2_client_credentials_connector", + "name": "OAuth2 Client Credentials Connector", + "type": "custom", + "description": "Generic OAuth2 connector for testing", + "version": "0.0.1", + "connector_params": [{"name": item} for item in secrets.keys()], + "client_config": { + "protocol": "https", + "host": secrets["domain"], + "authentication": { + "strategy": "oauth2_client_credentials", + "configuration": oauth2_client_credentials_configuration, + }, + }, + "endpoints": [], + "test_request": {"method": "GET", "path": "/test"}, + } + + fides_key = saas_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": secrets, + "saas_config": saas_config, + }, + ) + yield connection_config + connection_config.delete(db) + + @pytest.fixture(scope="session") def saas_config() -> Dict[str, Any]: saas_config = {} diff --git a/tests/ops/api/v1/endpoints/test_system.py b/tests/ops/api/v1/endpoints/test_system.py index 6bec60e07c..9aa855dc4a 100644 --- a/tests/ops/api/v1/endpoints/test_system.py +++ b/tests/ops/api/v1/endpoints/test_system.py @@ -112,13 +112,21 @@ def connections(): class TestPatchSystemConnections: @pytest.fixture(scope="function") - def system_linked_with_connection_config( + def system_linked_with_oauth2_authorization_code_connection_config( self, system: System, oauth2_authorization_code_connection_config, db: Session ): system.connection_configs = oauth2_authorization_code_connection_config db.commit() return system + @pytest.fixture(scope="function") + def system_linked_with_oauth2_client_credentials_connection_config( + self, system: System, oauth2_client_credentials_connection_config, db: Session + ): + system.connection_configs = oauth2_client_credentials_connection_config + db.commit() + return system + def test_patch_connections_valid_system( self, api_client: TestClient, generate_auth_header, url, payload ): @@ -205,12 +213,42 @@ def test_patch_connections_role_check_viewer( resp = api_client.patch(url, headers=auth_header, json=payload) assert resp.status_code == expected_status_code - def test_patch_connection_secrets_removes_access_token( + def test_patch_connection_secrets_removes_access_token_for_clients_credentials( + self, + api_client: TestClient, + generate_auth_header, + url, + system_linked_with_oauth2_client_credentials_connection_config, + ): + auth_header = generate_auth_header( + scopes=[CONNECTION_READ, CONNECTION_CREATE_OR_UPDATE] + ) + + # verify the connection_config is authorized + resp = api_client.get(url, headers=auth_header) + + assert resp.status_code == HTTP_200_OK + assert resp.json()["items"][0]["authorized"] is True + + # patch the connection_config with new secrets (but no access_token) + resp = api_client.patch( + f"{url}/secrets?verify=False", + headers=auth_header, + json={"domain": "test_domain"}, + ) + + # verify the connection_config is no longer authorized + resp = api_client.get(url, headers=auth_header) + + assert resp.status_code == HTTP_200_OK + assert resp.json()["items"][0]["authorized"] is False + + def test_patch_connection_secrets_removes_access_token_for_client_config( self, api_client: TestClient, generate_auth_header, url, - system_linked_with_connection_config, + system_linked_with_oauth2_authorization_code_connection_config, ): auth_header = generate_auth_header( scopes=[CONNECTION_READ, CONNECTION_CREATE_OR_UPDATE] @@ -429,7 +467,7 @@ def url(self, system) -> str: return V1_URL_PREFIX + f"/system/{system.fides_key}/connection" @pytest.fixture(scope="function") - def system_linked_with_connection_config( + def system_linked_with_oauth2_authorization_code_connection_config( self, system: System, connection_config, db: Session ): system.connection_configs = connection_config @@ -465,11 +503,13 @@ def test_delete_connection_config( api_client: TestClient, db: Session, generate_auth_header, - system_linked_with_connection_config, + system_linked_with_oauth2_authorization_code_connection_config, ) -> None: auth_header = generate_auth_header(scopes=[CONNECTION_DELETE]) # the key needs to be cached before the delete - key = system_linked_with_connection_config.connection_configs.key + key = ( + system_linked_with_oauth2_authorization_code_connection_config.connection_configs.key + ) resp = api_client.delete(url, headers=auth_header) assert resp.status_code == HTTP_204_NO_CONTENT assert db.query(ConnectionConfig).filter_by(key=key).first() is None @@ -576,13 +616,13 @@ def test_delete_connection_configs_role_viewer( acting_user_role, expected_status_code, assign_system, - system_linked_with_connection_config, + system_linked_with_oauth2_authorization_code_connection_config, request, db: Session, ) -> None: url = ( V1_URL_PREFIX - + f"/system/{system_linked_with_connection_config.fides_key}/connection" + + f"/system/{system_linked_with_oauth2_authorization_code_connection_config.fides_key}/connection" ) acting_user_role = request.getfixturevalue(acting_user_role) @@ -595,10 +635,12 @@ def test_delete_connection_configs_role_viewer( api_client.put( assign_url, headers=system_manager_auth_header, - json=[system_linked_with_connection_config.fides_key], + json=[ + system_linked_with_oauth2_authorization_code_connection_config.fides_key + ], ) auth_header = generate_system_manager_header( - [system_linked_with_connection_config.id] + [system_linked_with_oauth2_authorization_code_connection_config.id] ) else: auth_header = generate_role_header_for_user( @@ -622,13 +664,13 @@ def test_delete_connection_configs_role_check( generate_auth_header, acting_user_role, expected_status_code, - system_linked_with_connection_config, + system_linked_with_oauth2_authorization_code_connection_config, request, db: Session, ) -> None: url = ( V1_URL_PREFIX - + f"/system/{system_linked_with_connection_config.fides_key}/connection" + + f"/system/{system_linked_with_oauth2_authorization_code_connection_config.fides_key}/connection" ) acting_user_role = request.getfixturevalue(acting_user_role) diff --git a/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py b/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py index d3d19c3e0d..bb1755e33c 100644 --- a/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py +++ b/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py @@ -8,11 +8,6 @@ from sqlalchemy.orm import Session from fides.api.common_exceptions import FidesopsException, OAuth2TokenException -from fides.api.models.connectionconfig import ( - AccessLevel, - ConnectionConfig, - ConnectionType, -) from fides.api.service.authentication.authentication_strategy import ( AuthenticationStrategy, ) @@ -21,74 +16,6 @@ ) -@pytest.fixture(scope="function") -def oauth2_client_credentials_configuration() -> ( - OAuth2ClientCredentialsAuthenticationStrategy -): - return { - "token_request": { - "method": "POST", - "path": "/oauth/token", - "headers": [ - { - "name": "Content-Type", - "value": "application/x-www-form-urlencoded", - } - ], - "query_params": [ - {"name": "client_id", "value": ""}, - {"name": "client_secret", "value": ""}, - {"name": "grant_type", "value": "client_credentials"}, - ], - } - } - - -@pytest.fixture(scope="function") -def oauth2_client_credentials_connection_config( - db: Session, oauth2_client_credentials_configuration -) -> Generator: - secrets = { - "domain": "localhost", - "client_id": "client", - "client_secret": "secret", - "access_token": "access", - } - saas_config = { - "fides_key": "oauth2_client_credentials_connector", - "name": "OAuth2 Client Credentials Connector", - "type": "custom", - "description": "Generic OAuth2 connector for testing", - "version": "0.0.1", - "connector_params": [{"name": item} for item in secrets.keys()], - "client_config": { - "protocol": "https", - "host": secrets["domain"], - "authentication": { - "strategy": "oauth2_client_credentials", - "configuration": oauth2_client_credentials_configuration, - }, - }, - "endpoints": [], - "test_request": {"method": "GET", "path": "/test"}, - } - - fides_key = saas_config["fides_key"] - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": fides_key, - "name": fides_key, - "connection_type": ConnectionType.saas, - "access": AccessLevel.write, - "secrets": secrets, - "saas_config": saas_config, - }, - ) - yield connection_config - connection_config.delete(db) - - class TestAddAuthentication: # happy path, being able to use the existing access token def test_oauth2_authentication(