HJ-128 - FE&BE: Datamap export should only include the visible columns of the report

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
         description: Detect hardcoded secrets pre-commit using Gitleaks
         entry: gitleaks protect --verbose --no-banner --redact --staged
         language: system
-        stages: [commit]
+        stages: [pre-commit]
 
       - id: gitleaks-pre-push
         # To use this pre-push hook, run
@@ -20,4 +20,4 @@ repos:
         description: Detect hardcoded secrets pre-push using Gitleaks
         entry: gitleaks detect --verbose --no-banner --redact --log-opts "origin..HEAD"
         language: system
-        stages: [push]
+        stages: [pre-push]

clients/admin-ui/cypress/e2e/datamap-report.cy.ts

@@ -276,36 +276,36 @@ describe("Data map report table", () => {
       it("should cancel renaming columns", () => {
         cy.getByTestId("more-menu").click();
         cy.getByTestId("rename-columns-btn").click();
-        cy.getByTestId("column-data_use-input")
+        cy.getByTestId("column-data_uses-input")
           .clear()
           .then(() => {
-            cy.getByTestId("column-data_use-input").type("Custom Title");
+            cy.getByTestId("column-data_uses-input").type("Custom Title");
           });
         cy.getByTestId("rename-columns-cancel-btn").click({ force: true });
         cy.getByTestId("rename-columns-reset-btn").should("not.exist");
         cy.getByTestId("rename-columns-cancel-btn").should("not.exist");
         cy.getByTestId("rename-columns-apply-btn").should("not.exist");
-        cy.getByTestId("column-data_use").should("contain.text", "Data use");
+        cy.getByTestId("column-data_uses").should("contain.text", "Data use");
       });
       it("should reset columns", () => {
         cy.getByTestId("more-menu").click();
         cy.getByTestId("rename-columns-btn").click();
-        cy.getByTestId("column-data_use-input")
+        cy.getByTestId("column-data_uses-input")
           .clear()
           .then(() => {
-            cy.getByTestId("column-data_use-input").type("Custom Title");
+            cy.getByTestId("column-data_uses-input").type("Custom Title");
           });
         cy.getByTestId("rename-columns-apply-btn").click({ force: true });
         cy.getByTestId("more-menu").click();
         cy.getByTestId("rename-columns-btn").click();
         cy.getByTestId("rename-columns-reset-btn").click({ force: true });
-        cy.getByTestId("column-data_use").should("contain.text", "Data use");
+        cy.getByTestId("column-data_uses").should("contain.text", "Data use");
       });
       it("should support pressing the Enter key to apply renamed columns", () => {
         cy.getByTestId("more-menu").click();
         cy.getByTestId("rename-columns-btn").click();
-        cy.getByTestId("column-data_use-input").type("Custom Title{enter}");
-        cy.getByTestId("column-data_use").should(
+        cy.getByTestId("column-data_uses-input").type("Custom Title{enter}");
+        cy.getByTestId("column-data_uses").should(
           "contain.text",
           "Custom Title",
         );

clients/admin-ui/cypress/fixtures/custom-reports/custom-report.json

@@ -58,55 +58,22 @@
         "uses_non_cookie_access",
         "uses_profiling",
         "system_lorem"
-      ],
-      "columnVisibility": {
-        "dpo": true,
-        "egress": true,
-        "cookies": true,
-        "ingress": true,
-        "features": true,
-        "fides_key": true,
-        "legal_name": true,
-        "dpa_location": true,
-        "system_lorem": true,
-        "uses_cookies": true,
-        "data_stewards": true,
-        "data_subjects": false,
-        "legal_address": true,
-        "third_parties": true,
-        "cookie_refresh": true,
-        "privacy_policy": true,
-        "responsibility": true,
-        "uses_profiling": true,
-        "data_categories": true,
-        "declaration_name": true,
-        "retention_period": true,
-        "shared_categories": true,
-        "system_dependencies": true,
-        "reason_for_exemption": true,
-        "joint_controller_info": true,
-        "cookie_max_age_seconds": true,
-        "uses_non_cookie_access": true,
-        "data_security_practices": true,
-        "processes_personal_data": true,
-        "third_country_safeguards": true,
-        "administrating_department": true,
-        "legal_basis_for_profiling": true,
-        "legal_basis_for_transfers": true,
-        "impact_assessment_location": true,
-        "legal_basis_for_processing": true,
-        "link_to_processor_contract": true,
-        "does_international_transfers": true,
-        "special_category_legal_basis": true,
-        "DATA_SHARED_WITH_THIRD_PARTIES": true,
-        "exempt_from_privacy_regulations": true,
-        "system_undeclared_data_categories": false,
-        "legitimate_interest_disclosure_url": true,
-        "data_use_undeclared_data_categories": false,
-        "flexible_legal_basis_for_processing": true,
-        "requires_data_protection_assessments": true
-      }
+      ]
     },
-    "column_map": { "data_categories": "My Data Category Map" }
+    "column_map": {
+      "data_categories": {
+        "label": "My Data Category Map",
+        "enabled": true
+      },
+      "data_subjects": {
+        "enabled": false
+      },
+      "system_undeclared_data_categories": {
+        "enabled": false
+      },
+      "data_use_undeclared_data_categories": {
+        "enabled": false
+      }
+    }
   }
 }

clients/admin-ui/src/features/common/custom-reports/CustomReportCreationModal.tsx

@@ -26,7 +26,7 @@ interface CustomReportCreationModalProps {
   isOpen: boolean;
   handleClose: () => void;
   tableStateToSave: CustomReportTableState | undefined;
-  columnMapToSave: Record<string, string> | undefined;
+  columnMapToSave: Record<string, Record<string, any>> | undefined;
   unavailableNames?: string[];
   onCreateCustomReport: (newReport: CustomReportResponse) => void;
 }
@@ -35,7 +35,7 @@ export const CustomReportCreationModal = ({
   isOpen,
   handleClose,
   tableStateToSave,
-  columnMapToSave,
+  columnMapToSave = {},
   unavailableNames,
   onCreateCustomReport,
 }: CustomReportCreationModalProps) => {

clients/admin-ui/src/features/common/custom-reports/CustomReportTemplates.tsx

@@ -54,7 +54,7 @@ interface CustomReportTemplatesProps {
   reportType: ReportType;
   savedReportId: string; // from local storage
   tableStateToSave: CustomReportTableState | undefined;
-  currentColumnMap?: Record<string, string> | undefined;
+  currentColumnMap?: Record<string, Record<string, any>> | undefined;
   onCustomReportSaved: (customReport: CustomReportResponse | null) => void;
   onSavedReportDeleted: () => void;
 }

clients/admin-ui/src/features/datamap/datamap.slice.ts

@@ -108,6 +108,7 @@ const datamapApi = baseApi.injectEndpoints({
         dataSubjects?: string;
         format?: ExportFormat;
         report_id?: string;
+        report?: object;
       }
     >({
       query: ({
@@ -120,6 +121,7 @@ const datamapApi = baseApi.injectEndpoints({
         dataSubjects,
         format,
         report_id,
+        report,
       }) => {
         let queryString = `page=${pageIndex}&size=${pageSize}&group_by=${groupBy}`;
         if (dataUses) {
@@ -139,6 +141,8 @@ const datamapApi = baseApi.injectEndpoints({
         }
         return {
           url: `plus/datamap/minimal/${format}?${queryString}`,
+          method: "POST",
+          body: { report },
           responseHandler: async (response) => {
             const filename = await getFileNameFromContentDisposition(
               response.headers.get("content-disposition"),

clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx

@@ -195,7 +195,9 @@ export const DatamapReportTable = () => {
 
   // Column renaming
   const [isRenamingColumns, setIsRenamingColumns] = useState(false);
-  const handleColumnRenaming = (values: Record<string, string>) => {
+  const handleColumnRenaming = (
+    values: Record<string, Record<string, any>>,
+  ) => {
     setSavedCustomReportId("");
     setColumnNameMapOverrides(values);
     setIsRenamingColumns(false);
@@ -239,6 +241,18 @@ export const DatamapReportTable = () => {
       ...reportQuery,
       format: downloadType,
       report_id: savedCustomReportId,
+      report: {
+        name: "temporary report",
+        type: "datamap",
+        config: {
+          column_map: columnNameMapOverrides,
+          table_state: {
+            groupBy,
+            filters: selectedFilters,
+            columnOrder,
+          },
+        },
+      },
     }).then(() => {
       if (isExportReportSuccess) {
         onExportReportClose();
@@ -308,7 +322,7 @@ export const DatamapReportTable = () => {
   const handleSavedReport = (
     savedReport: CustomReportResponse | null,
     resetForm: (
-      nextState?: Partial<FormikState<Record<string, string>>> | undefined,
+      nextState?: Partial<FormikState<Record<string, any>>> | undefined,
     ) => void,
   ) => {
     if (!savedReport) {

clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx

@@ -20,29 +20,38 @@ const CUSTOM_FIELD_SYSTEM_PREFIX = "system_";
 const CUSTOM_FIELD_DATA_USE_PREFIX = "privacy_declaration_";
 
 export const getDefaultColumn: (
-  columnNameMap: Record<string, string>,
+  columnNameMap: Record<string, any>,
   isRenamingColumns: boolean,
 ) => Partial<ColumnDef<DatamapReport>> = (
   columnNameMap,
   isRenamingColumns,
 ) => ({
   cell: (props) => <DefaultCell value={props.getValue() as string} />,
-  header: (props) => (
-    <EditableHeaderCell
-      value={getColumnHeaderText({
-        columnId: props.column.id,
-        columnNameMap,
-      })}
-      defaultValue={
-        DEFAULT_COLUMN_NAMES[props.column.id as COLUMN_IDS] ||
-        getColumnHeaderText({
-          columnId: props.column.id,
-        })
+  header: (props) => {
+    const newColumnNameMap: Record<string, string> = {};
+    Object.keys(columnNameMap).forEach((key) => {
+      newColumnNameMap[key] = columnNameMap[key];
+      if (columnNameMap[key].label) {
+        newColumnNameMap[key] = columnNameMap[key].label;
       }
-      isEditing={isRenamingColumns}
-      {...props}
-    />
-  ),
+    });
+    return (
+      <EditableHeaderCell
+        value={getColumnHeaderText({
+          columnId: props.column.id,
+          columnNameMap: newColumnNameMap,
+        })}
+        defaultValue={
+          DEFAULT_COLUMN_NAMES[props.column.id as COLUMN_IDS] ||
+          getColumnHeaderText({
+            columnId: props.column.id,
+          })
+        }
+        isEditing={isRenamingColumns}
+        {...props}
+      />
+    );
+  },
 });
 
 const getCustomFieldColumns = (

clients/admin-ui/src/features/datamap/reporting/constants.tsx

@@ -15,7 +15,7 @@ export enum DATAMAP_LOCAL_STORAGE_KEYS {
 
 export enum COLUMN_IDS {
   SYSTEM_NAME = "system_name",
-  DATA_USE = "data_use",
+  DATA_USE = "data_uses",
   DATA_CATEGORY = "data_categories",
   DATA_SUBJECT = "data_subjects",
   LEGAL_NAME = "legal_name",

clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx

@@ -26,8 +26,10 @@ interface DatamapReportContextProps {
   setColumnVisibility: Dispatch<SetStateAction<Record<string, boolean>>>;
   columnSizing: Record<string, number>;
   setColumnSizing: Dispatch<SetStateAction<Record<string, number>>>;
-  columnNameMapOverrides: Record<string, string>;
-  setColumnNameMapOverrides: Dispatch<SetStateAction<Record<string, string>>>;
+  columnNameMapOverrides: Record<string, Record<string, any>>;
+  setColumnNameMapOverrides: Dispatch<
+    SetStateAction<Record<string, Record<string, any>>>
+  >;
 }
 
 export const DatamapReportContext = createContext<DatamapReportContextProps>(
@@ -76,7 +78,7 @@ export const DatamapReportProvider = ({
   >(DATAMAP_LOCAL_STORAGE_KEYS.COLUMN_SIZING, {});
 
   const [columnNameMapOverrides, setColumnNameMapOverrides] = useLocalStorage<
-    Record<string, string>
+    Record<string, Record<string, any>>
   >(DATAMAP_LOCAL_STORAGE_KEYS.COLUMN_NAMES, {});
 
   const contextValue: DatamapReportContextProps = useMemo(

clients/admin-ui/src/types/api/models/CustomReportConfig.ts

@@ -13,5 +13,5 @@ export type CustomReportConfig = {
   /**
    * A map between column keys and custom labels
    */
-  column_map?: Record<string, string>;
+  column_map?: Record<string, any>;
 };

src/fides/api/schemas/custom_report.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Any, Dict
+from typing import Any, Dict, Optional, Set
 
 from pydantic import Field
 
@@ -13,13 +13,40 @@ class ReportType(str, Enum):
     privacy_request = "privacy_request"
 
 
+class ColumnMapItem(FidesSchema):
+    """A map between column keys and custom labels."""
+
+    label: Optional[str] = Field(
+        default=None, description="The custom label for the column"
+    )
+    enabled: Optional[bool] = Field(
+        default=None, description="Whether the column is shown"
+    )
+
+
 class CustomReportConfig(FidesSchema):
     """The configuration for a custom report."""
 
     table_state: Dict[str, Any] = Field(
         default_factory=dict,
         description="Flexible dictionary storing UI-specific table state data without a fixed schema",
     )
-    column_map: Dict[str, str] = Field(
+    column_map: Dict[str, ColumnMapItem] = Field(
         default_factory=dict, description="A map between column keys and custom labels"
     )
+
+    @property
+    def columns_to_skip(self) -> Set[str]:
+        return {
+            key
+            for key, value in self.column_map.items()  # pylint: disable=no-member
+            if value.enabled is False
+        }
+
+    @property
+    def custom_column_labels(self) -> Dict[str, str]:
+        return {
+            key: value.label
+            for key, value in self.column_map.items()  # pylint: disable=no-member
+            if value.label
+        }