Track and report system changes (#3990)

.fides/db_dataset.yml

@@ -1078,6 +1078,9 @@ dataset:
     - name: data_security_practices
       data_categories:
       - system.operations
+    - name: user_id
+      data_categories:
+      - user.unique_id
   - name: messagingconfig
     description: 'Fides Generated Description for Table: messagingconfig'
     data_categories: []
@@ -2553,4 +2556,29 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
     - name: updated_at
       data_categories: [system.operations]
-      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+  - name: plus_system_history
+    description: 'Table used to store system changes'
+    data_categories: []
+    data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    fields:
+    - name: id
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: system_id
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: user_id
+      data_categories: [user.unique_id]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: before
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: after
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: created_at
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: updated_at
+      data_categories: [system.operations]

src/fides/api/alembic/migrations/versions/093bb28a8270_add_plus_system_history_table.py

@@ -0,0 +1,63 @@
+"""Add plus_system_history table
+
+Revision ID: 093bb28a8270
+Revises: 3038667ba898
+Create Date: 2023-08-18 23:48:22.934916
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "093bb28a8270"
+down_revision = "3038667ba898"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "plus_system_history",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("user_id", sa.String(), nullable=True),
+        sa.Column("system_id", sa.String(), nullable=False),
+        sa.Column("before", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.Column("after", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.ForeignKeyConstraint(["system_id"], ["ctl_systems.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_plus_system_history_id"), "plus_system_history", ["id"], unique=False
+    )
+    op.create_index(
+        "idx_plus_system_history_created_at_system_id",
+        "plus_system_history",
+        ["created_at", "system_id"],
+    )
+    op.add_column(
+        "ctl_systems",
+        sa.Column("user_id", sa.String, nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_column("ctl_systems", "user_id")
+    op.drop_index(
+        op.f("idx_plus_system_history_created_at_system_id"),
+        table_name="plus_system_history",
+    )
+    op.drop_index(op.f("ix_plus_system_history_id"), table_name="plus_system_history")
+    op.drop_table("plus_system_history")

src/fides/api/api/v1/endpoints/system.py

@@ -29,13 +29,14 @@
     validate_privacy_declarations,
 )
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType
-from fides.api.models.sql_models import System  # type: ignore[attr-defined]
+from fides.api.models.fides_user import FidesUser
+from fides.api.models.sql_models import System  # type:ignore[attr-defined]
 from fides.api.oauth.system_manager_oauth_util import (
     verify_oauth_client_for_system_from_fides_key,
     verify_oauth_client_for_system_from_fides_key_cli,
     verify_oauth_client_for_system_from_request_body_cli,
 )
-from fides.api.oauth.utils import verify_oauth_client_prod
+from fides.api.oauth.utils import get_current_user, verify_oauth_client_prod
 from fides.api.schemas.connection_configuration import connection_secrets_schemas
 from fides.api.schemas.connection_configuration.connection_config import (
     BulkPutConnectionConfiguration,
@@ -247,13 +248,14 @@ async def update(
     ),  # Security dependency defined here instead of the path operation decorator so we have access to the request body
     # to be able to look up the system as well as return a value
     db: AsyncSession = Depends(get_async_db),
+    current_user: FidesUser = Depends(get_current_user),
 ) -> Dict:
     """
     Update a System by the fides_key extracted from the request body.  Defined outside of the crud routes
     to add additional "system manager" permission checks.
     """
     await validate_privacy_declarations(db, resource)
-    return await update_system(resource, db)
+    return await update_system(resource, db, current_user.id if current_user else None)
 
 
 @SYSTEM_ROUTER.post(
@@ -272,8 +274,11 @@ async def upsert(
     resources: List[SystemSchema],
     response: Response,
     db: AsyncSession = Depends(get_async_db),
+    current_user: FidesUser = Depends(get_current_user),
 ) -> Dict:
-    inserted, updated = await upsert_system(resources, db)
+    inserted, updated = await upsert_system(
+        resources, db, current_user.id if current_user else None
+    )
     response.status_code = (
         status.HTTP_201_CREATED if inserted > 0 else response.status_code
     )
@@ -339,12 +344,13 @@ async def delete(
 async def create(
     resource: SystemSchema,
     db: AsyncSession = Depends(get_async_db),
+    current_user: FidesUser = Depends(get_current_user),
 ) -> Dict:
     """
     Override `System` create/POST to handle `.privacy_declarations` defined inline,
     for backward compatibility and ease of use for API users.
     """
-    return await create_system(resource, db)
+    return await create_system(resource, db, current_user.id if current_user else None)
 
 
 @SYSTEM_ROUTER.get(

src/fides/api/db/base.py

@@ -34,4 +34,5 @@
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.models.registration import UserRegistration
 from fides.api.models.storage import StorageConfig
+from fides.api.models.system_history import SystemHistory
 from fides.api.models.system_manager import SystemManager

src/fides/api/db/system.py

@@ -1,8 +1,11 @@
 """
 Functions for interacting with System objects in the database.
 """
-from typing import Dict, List, Optional, Tuple
+import copy
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
 
+from deepdiff import DeepDiff
 from fastapi import HTTPException
 from fideslang.models import Cookies as CookieSchema
 from fideslang.models import System as SystemSchema
@@ -19,6 +22,7 @@
     PrivacyDeclaration,
     System,
 )
+from fides.api.models.system_history import SystemHistory
 from fides.api.util.errors import NotFoundError
 
 
@@ -37,7 +41,7 @@ def get_system(db: Session, fides_key: str) -> System:
     if system is None:
         raise HTTPException(
             status_code=HTTP_404_NOT_FOUND,
-            detail="A valid system must be provided to create, update, and delete connections",
+            detail="The specified system was not found. Please provide a valid system for the requested operation.",
         )
     return system
 
@@ -74,7 +78,9 @@ async def validate_privacy_declarations(db: AsyncSession, system: SystemSchema)
 
 
 async def upsert_system(
-    resources: List[SystemSchema], db: AsyncSession
+    resources: List[SystemSchema],
+    db: AsyncSession,
+    current_user_id: Optional[str] = None,
 ) -> Tuple[int, int]:
     """Helper method to abstract system upsert logic from API code"""
     inserted = 0
@@ -90,10 +96,12 @@ async def upsert_system(
             log.debug(
                 f"Upsert System with fides_key {resource.fides_key} not found, will create"
             )
-            await create_system(resource=resource, db=db)
+            await create_system(
+                resource=resource, db=db, current_user_id=current_user_id
+            )
             inserted += 1
             continue
-        await update_system(resource=resource, db=db)
+        await update_system(resource=resource, db=db, current_user_id=current_user_id)
         updated += 1
     return (inserted, updated)
 
@@ -197,11 +205,14 @@ async def upsert_cookies(
     )
 
 
-async def update_system(resource: SystemSchema, db: AsyncSession) -> Dict:
+async def update_system(
+    resource: SystemSchema, db: AsyncSession, current_user_id: Optional[str] = None
+) -> Dict:
     """Helper function to share core system update logic for wrapping endpoint functions"""
     system: System = await get_resource(
         sql_model=System, fides_key=resource.fides_key, async_session=db
     )
+    existing_system_dict = copy.deepcopy(SystemSchema.from_orm(system).dict())
 
     # handle the privacy declaration upsert logic
     try:
@@ -220,12 +231,81 @@ async def update_system(resource: SystemSchema, db: AsyncSession) -> Dict:
     updated_system = await update_resource(System, resource.dict(), db)
     async with db.begin():
         await db.refresh(updated_system)
+        _audit_system_changes(
+            db,
+            system.id,
+            current_user_id,
+            existing_system_dict,
+            SystemSchema.from_orm(updated_system).dict(),
+        )
+
     return updated_system
 
 
+def _audit_system_changes(
+    db: Session,
+    system_id: str,
+    current_user_id: Optional[str],
+    existing_system: Dict[str, Any],
+    updated_system: Dict[str, Any],
+) -> None:
+    """
+    Audits changes made to a system and logs them in the SystemHistory table.
+    The function creates separate SystemHistory entries for general changes,
+    changes to privacy declarations (data uses), and changes to egress and ingress (data flow) settings.
+    This is done to match the way the user interacts with the system from the UI.
+    """
+
+    # Extract egress, ingress, and privacy_declarations fields
+    egress_ingress_existing = {
+        field: existing_system.pop(field, None) for field in ["egress", "ingress"]
+    }
+    egress_ingress_updated = {
+        field: updated_system.pop(field, None) for field in ["egress", "ingress"]
+    }
+    privacy_existing = {
+        "privacy_declarations": existing_system.pop("privacy_declarations", [])
+    }
+    privacy_updated = {
+        "privacy_declarations": updated_system.pop("privacy_declarations", [])
+    }
+
+    # Get the current datetime
+    now = datetime.now()
+
+    # Create a SystemHistory entry for general changes
+    if DeepDiff(existing_system, updated_system, ignore_order=True):
+        SystemHistory(
+            user_id=current_user_id,
+            system_id=system_id,
+            before=existing_system,
+            after=updated_system,
+            created_at=now,
+        ).save(db=db)
+
+    # Create a SystemHistory entry for changes to privacy_declarations
+    if DeepDiff(privacy_existing, privacy_updated, ignore_order=True):
+        SystemHistory(
+            user_id=current_user_id,
+            system_id=system_id,
+            before=privacy_existing,
+            after=privacy_updated,
+            created_at=now,
+        ).save(db=db)
+
+    # Create a SystemHistory entry for changes to egress and ingress
+    if DeepDiff(egress_ingress_existing, egress_ingress_updated, ignore_order=True):
+        SystemHistory(
+            user_id=current_user_id,
+            system_id=system_id,
+            before=egress_ingress_existing,
+            after=egress_ingress_updated,
+            created_at=now,
+        ).save(db=db)
+
+
 async def create_system(
-    resource: SystemSchema,
-    db: AsyncSession,
+    resource: SystemSchema, db: AsyncSession, current_user_id: Optional[str] = None
 ) -> Dict:
     """
     Override `System` create/POST to handle `.privacy_declarations` defined inline,
@@ -241,8 +321,13 @@ async def create_system(
 
     # create the system resource using generic creation
     # the system must be created before the privacy declarations so that it can be referenced
+    resource_dict = resource.dict()
+
+    # set the current user's ID
+    resource_dict["user_id"] = current_user_id
+
     created_system = await create_resource(
-        System, resource_dict=resource.dict(), async_session=db
+        System, resource_dict=resource_dict, async_session=db
     )
 
     privacy_declaration_exception = None

src/fides/api/models/sql_models.py

@@ -37,7 +37,9 @@
 from fides.api.models.client import ClientDetail
 from fides.api.models.fides_user import FidesUser
 from fides.api.models.fides_user_permissions import FidesUserPermissions
-from fides.config import CONFIG
+from fides.config import get_config
+
+CONFIG = get_config()
 
 
 class FidesBase(FideslibBase):
@@ -401,6 +403,8 @@ class System(Base, FidesBase):
         "Cookies", back_populates="system", lazy="selectin", uselist=True, viewonly=True
     )
 
+    user_id = Column(String, nullable=True)
+
     @classmethod
     def get_data_uses(
         cls: Type[System], systems: List[System], include_parents: bool = True