From ee996208b8ae6d2a4516fe0963d7055b6a3ac989 Mon Sep 17 00:00:00 2001 From: Thomas La Piana Date: Tue, 24 Aug 2021 18:26:14 +0000 Subject: [PATCH] adds very basic diff functionality --- fidesctl/data/sample/system.yml | 6 +- fidesctl/requirements.txt | 1 + fidesctl/src/fidesctl.egg-info/requires.txt | 1 + fidesctl/src/fidesctl/cli/cli.py | 8 ++- fidesctl/src/fidesctl/cli/utils.py | 7 +- fidesctl/src/fidesctl/core/apply.py | 77 +++++++++++++++------ 6 files changed, 72 insertions(+), 28 deletions(-) diff --git a/fidesctl/data/sample/system.yml b/fidesctl/data/sample/system.yml index f7291c1a3f..d4b37f7199 100644 --- a/fidesctl/data/sample/system.yml +++ b/fidesctl/data/sample/system.yml @@ -1,6 +1,6 @@ system: - organizationId: 1 - fidesKey: "dataAnalyticsSystem" + fidesKey: "dataAnalyticSystem" name: "Data Analytics System" description: "A system used for analyzing customer behaviour." systemType: "Service" @@ -18,7 +18,7 @@ system: - organizationId: 1 fidesKey: "customerDataSharingSystem" - name: "Customer Data Sharing System" + name: "Customer Data Sharin System" description: "Share data about our users with third-parties." systemType: "Service" privacyDeclarations: @@ -29,4 +29,6 @@ system: dataQualifier: "identified_data" dataSubjects: - "customer" + datasetReferences: + - "sample_db_dataset.Email" systemDependencies: [] diff --git a/fidesctl/requirements.txt b/fidesctl/requirements.txt index 34bc895bc0..a73578e5d7 100644 --- a/fidesctl/requirements.txt +++ b/fidesctl/requirements.txt @@ -1,5 +1,6 @@ click==7.1.2 colorama==0.4.4 +deepdiff==5.5.0 mysql-connector-python==8.0.24 PyJWT==2.1.0 psycopg2-binary==2.8.6 diff --git a/fidesctl/src/fidesctl.egg-info/requires.txt b/fidesctl/src/fidesctl.egg-info/requires.txt index 34bc895bc0..a73578e5d7 100644 --- a/fidesctl/src/fidesctl.egg-info/requires.txt +++ b/fidesctl/src/fidesctl.egg-info/requires.txt @@ -1,5 +1,6 @@ click==7.1.2 colorama==0.4.4 +deepdiff==5.5.0 mysql-connector-python==8.0.24 PyJWT==2.1.0 psycopg2-binary==2.8.6 diff --git a/fidesctl/src/fidesctl/cli/cli.py b/fidesctl/src/fidesctl/cli/cli.py index e0c6995d09..b03fe2dd62 100644 --- a/fidesctl/src/fidesctl/cli/cli.py +++ b/fidesctl/src/fidesctl/cli/cli.py @@ -114,8 +114,13 @@ def show(ctx: click.Context, object_type: str) -> None: is_flag=True, help="Runs the apply command without any side-effects.", ) +@click.option( + "--diff", + is_flag=True, + help="Outputs a detailed diff of the local resource files compared to the server resources.", +) @click.argument("manifest_dir", type=click.Path()) -def apply(ctx: click.Context, dry: bool, manifest_dir: str) -> None: +def apply(ctx: click.Context, dry: bool, diff: bool, manifest_dir: str) -> None: """ Send the manifest files to the server. """ @@ -125,6 +130,7 @@ def apply(ctx: click.Context, dry: bool, manifest_dir: str) -> None: manifests_dir=manifest_dir, headers=config.user.request_headers, dry=dry, + diff=diff, ) diff --git a/fidesctl/src/fidesctl/cli/utils.py b/fidesctl/src/fidesctl/cli/utils.py index 7991c3808e..db78ed6053 100644 --- a/fidesctl/src/fidesctl/cli/utils.py +++ b/fidesctl/src/fidesctl/cli/utils.py @@ -13,10 +13,13 @@ def pretty_echo(dict_object: Dict, color: str = "white") -> None: click.secho(json.dumps(dict_object, indent=2), fg=color) -def handle_cli_response(response: requests.Response) -> requests.Response: +def handle_cli_response( + response: requests.Response, verbose: bool = True +) -> requests.Response: """Viewable CLI response""" if response.status_code >= 200 and response.status_code <= 299: - pretty_echo(response.json(), "green") + if verbose: + pretty_echo(response.json(), "green") else: try: pretty_echo(response.json(), "red") diff --git a/fidesctl/src/fidesctl/core/apply.py b/fidesctl/src/fidesctl/core/apply.py index 280bd2035f..6dfce2c87b 100644 --- a/fidesctl/src/fidesctl/core/apply.py +++ b/fidesctl/src/fidesctl/core/apply.py @@ -1,6 +1,8 @@ """This module handles the logic required for applying manifest files to the server.""" from typing import Dict, List, Tuple, Optional, Iterable +from deepdiff import DeepDiff + from fidesctl.core import api, manifests, parse from fidesctl.core.models import FidesModel from fidesctl.cli.utils import handle_cli_response @@ -8,11 +10,16 @@ def sort_create_update_unchanged( - manifest_object_list: List[FidesModel], server_object_list: List[FidesModel] + manifest_object_list: List[FidesModel], + server_object_list: List[FidesModel], + diff: bool = False, ) -> Tuple[List[FidesModel], List[FidesModel], List[FidesModel]]: """ Check the contents of the object lists and populate separate new lists for object creation, updating, or no change. + + The `diff` flag will print out the differences between the server objects + the local resource files. """ server_object_dict = { server_object.fidesKey: server_object for server_object in server_object_list @@ -33,9 +40,13 @@ def sort_create_update_unchanged( if manifest_object == server_object: unchanged_list.append(manifest_object) else: + if diff: + print(DeepDiff(server_object, manifest_object)) update_list.append(manifest_object) else: + if diff: + print(manifest_object) create_list.append(manifest_object) return create_list, update_list, unchanged_list @@ -64,7 +75,8 @@ def execute_create_update_unchanged( headers=headers, object_type=object_type, json_object=create_object.json(exclude_none=True), - ) + ), + verbose=False, ) echo_green(success_echo.format("Created", object_type, create_object.fidesKey)) for update_object in update_list: @@ -75,7 +87,8 @@ def execute_create_update_unchanged( object_type=object_type, object_id=update_object.id, json_object=update_object.json(exclude_none=True), - ) + ), + verbose=False, ) echo_green(success_echo.format("Updated", object_type, update_object.fidesKey)) for unchanged_object in unchanged_list: @@ -104,23 +117,45 @@ def get_server_objects( return server_object_list -def apply( - url: str, manifests_dir: str, headers: Dict[str, str], dry: bool = False -) -> None: +def echo_results(action: str, object_type: str, resource_list: List) -> None: """ - Apply the current manifest file state to the server. - Excludes systems and registries. + Echo out the results of the apply. """ - ingested_manifests = manifests.ingest_manifests(manifests_dir) + echo_green(f"{action.upper()} {len(resource_list)} {object_type} objects.") + - parsed_manifests: Dict[str, List[FidesModel]] = { +def parse_manifests( + raw_manifests: Dict[str, List[Dict]] +) -> Dict[str, List[FidesModel]]: + """ + Parse the raw resource manifests into resource objects. + """ + parsed_manifests = { object_type: [ parse.parse_manifest(object_type, _object) for _object in object_list ] - for object_type, object_list in ingested_manifests.items() + for object_type, object_list in raw_manifests.items() } + return parsed_manifests + + +def apply( + url: str, + manifests_dir: str, + headers: Dict[str, str], + dry: bool = False, + diff: bool = False, +) -> None: + """ + Apply the current manifest file state to the server. + Excludes systems and registries. + """ + ingested_manifests = manifests.ingest_manifests(manifests_dir) + parsed_manifests = parse_manifests(ingested_manifests) for object_type, manifest_object_list in parsed_manifests.items(): + # Doing some echos here to make a pretty output + echo_green("-" * 10) existing_keys = [ manifest_object.fidesKey for manifest_object in manifest_object_list @@ -131,16 +166,13 @@ def apply( # Determine which objects should be created, updated, or are unchanged create_list, update_list, unchanged_list = sort_create_update_unchanged( - manifest_object_list, - server_object_list, + manifest_object_list, server_object_list, diff ) if dry: - echo_green(f"Would Create {len(create_list)} {object_type} objects.") - echo_green(f"Would Update {len(update_list)} {object_type} objects.") - echo_green( - f"Would Skip {len(unchanged_list)} unchanged {object_type} objects." - ) + echo_results("would create", object_type, create_list) + echo_results("would update", object_type, update_list) + echo_results("would skip", object_type, unchanged_list) else: execute_create_update_unchanged( url, @@ -151,8 +183,7 @@ def apply( unchanged_list, ) - echo_green(f"Created {len(create_list)} {object_type} objects.") - echo_green(f"Updated {len(update_list)} {object_type} objects.") - echo_green( - f"Skipped {len(unchanged_list)} unchanged {object_type} objects." - ) + echo_results("created", object_type, create_list) + echo_results("updated", object_type, update_list) + echo_results("skipped", object_type, unchanged_list) + echo_green("-" * 10)