Fides Pydantic V2 Upgrade (#5020)

Pydantic v1 -> Pydantic v2 upgrade

Co-authored-by: Adrian Galvan <[email protected]>
Co-authored-by: eastandwestwind <[email protected]>

.github/workflows/backend_checks.yml

@@ -43,7 +43,7 @@ jobs:
     strategy:
       matrix:
         # NOTE: These are the currently supported/tested Python Versions
-        python_version: ["3.8.18", "3.9.18", "3.10.13"]
+        python_version: ["3.9.18", "3.10.13"]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -223,7 +223,7 @@ jobs:
     needs: Check-Container-Startup
     strategy:
       matrix:
-        python_version: ["3.8.18", "3.9.18", "3.10.13"]
+        python_version: ["3.9.18", "3.10.13"]
         test_selection:
           - "ctl-not-external"
           - "ops-unit-api"
@@ -275,7 +275,7 @@ jobs:
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
       matrix:
-        python_version: ["3.8.18", "3.9.18", "3.10.13"]
+        python_version: ["3.9.18", "3.10.13"]
     runs-on: ubuntu-latest
     timeout-minutes: 20
     # In PRs run with the "unsafe" label, or run on a "push" event to main
@@ -315,7 +315,7 @@ jobs:
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
       matrix:
-        python_version: ["3.8.18", "3.9.18", "3.10.13"]
+        python_version: ["3.9.18", "3.10.13"]
     runs-on: ubuntu-latest
     timeout-minutes: 20
     # In PRs run with the "unsafe" label, or run on a "push" event to main
@@ -381,7 +381,7 @@ jobs:
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
       matrix:
-        python_version: ["3.8.18", "3.9.18", "3.10.13"]
+        python_version: ["3.9.18", "3.10.13"]
     steps:
       - name: Download container
         uses: actions/download-artifact@v4

CHANGELOG.md

@@ -17,6 +17,9 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fides/compare/2.43.0...main)
 
+### Added
+- Pydantic v1 -> Pydantic v2 upgrade [#5020](https://github.com/ethyca/fides/pull/5020)
+
 ### Fixed
 - Ignore `404` errors on Oracle Responsys deletions [#5203](https://github.com/ethyca/fides/pull/5203)
 - Fix white screen issue when privacy request has null value for daysLeft [#5213](https://github.com/ethyca/fides/pull/5213)

README.md

@@ -25,7 +25,7 @@ In order to get started quickly with Fides, a sample project is bundled within t
 #### Minimum requirements (for all platforms)
 
 * [Docker](https://www.docker.com/products/docker-desktop) (version 20.10.11 or later)
-* [Python](https://www.python.org/downloads/) (version 3.8 through 3.10)
+* [Python](https://www.python.org/downloads/) (version 3.9 through 3.10)
 
 #### Download and install Fides
 

clients/admin-ui/src/features/common/ConnectedCircle.tsx

@@ -3,7 +3,7 @@ import { Box, BoxProps } from "fidesui";
 const ConnectedCircle = ({
   connected,
   ...other
-}: { connected?: boolean } & BoxProps) => {
+}: { connected?: boolean | null } & BoxProps) => {
   let color = "red.500";
   if (connected == null) {
     color = "gray.300";

clients/admin-ui/src/features/common/SystemsCheckboxTable.tsx

@@ -54,7 +54,7 @@ export const SystemTableCell = ({
         whiteSpace="nowrap"
         overflow="hidden"
         textOverflow="ellipsis"
-        title={system.fidesctl_meta?.resource_id}
+        title={system.fidesctl_meta?.resource_id || ""}
       >
         {system.fidesctl_meta?.resource_id}
       </Box>

clients/admin-ui/src/features/common/form/inputs.tsx

@@ -64,7 +64,7 @@ type Variant = "inline" | "stacked" | "block";
 export interface CustomInputProps {
   disabled?: boolean;
   label?: string;
-  tooltip?: string;
+  tooltip?: string | null;
   variant?: Variant;
   isRequired?: boolean;
   textColor?: string;
@@ -172,7 +172,7 @@ const ClearIndicator = () => null;
 export interface Option {
   value: string;
   label: string;
-  description?: string;
+  description?: string | null;
   tooltip?: string;
 }
 
@@ -204,8 +204,8 @@ export interface SelectProps {
   label?: string;
   labelProps?: FormLabelProps;
   placeholder?: string;
-  tooltip?: string;
-  options: Option[];
+  tooltip?: string | null;
+  options?: Option[] | [];
   isDisabled?: boolean;
   isSearchable?: boolean;
   isClearable?: boolean;
@@ -247,7 +247,7 @@ export const SELECT_STYLES: ChakraStylesConfig<
 };
 
 export const SelectInput = ({
-  options,
+  options = [],
   fieldName,
   placeholder,
   size,
@@ -399,7 +399,7 @@ interface CreatableSelectProps extends SelectProps {
   disableMenu?: boolean;
 }
 const CreatableSelectInput = ({
-  options,
+  options = [],
   placeholder,
   fieldName,
   size,

clients/admin-ui/src/features/common/hooks/usePicker.ts

@@ -51,7 +51,7 @@ interface UsePaginatedPickerProps {
   initialSelected: string[];
   initialExcluded: string[];
   initialAllSelected?: boolean;
-  itemCount: number;
+  itemCount: number | null;
 }
 
 export const usePaginatedPicker = ({

clients/admin-ui/src/features/common/hooks/useTaxonomies.tsx

@@ -35,8 +35,8 @@ const useTaxonomies = () => {
     fidesLangKey: string,
     getDataFunction: (fidesLangKey: string) =>
       | {
-          parent_key?: string;
-          name?: string;
+          parent_key?: string | null;
+          name?: string | null;
         }
       | undefined,
     primaryLevel = 1,

clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordion.tsx

@@ -1,7 +1,7 @@
 import { Accordion } from "fidesui";
 import React from "react";
 
-import { System } from "~/types/api/models/System";
+import { System } from "~/types/api";
 
 import { DataFlowAccordionForm } from "./DataFlowAccordionForm";
 

clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx

@@ -45,7 +45,7 @@ export const useServerSidePagination = () => {
   const defaultPageIndex = 1;
   const [pageSize, setPageSize] = useState(PAGE_SIZES[0]);
   const [pageIndex, setPageIndex] = useState<number>(defaultPageIndex);
-  const [totalPages, setTotalPages] = useState<number>();
+  const [totalPages, setTotalPages] = useState<number | null>();
   const onPreviousPageClick = useCallback(() => {
     setPageIndex((prev) => prev - 1);
   }, [setPageIndex]);

clients/admin-ui/src/features/common/table/v2/cells.tsx

@@ -27,7 +27,7 @@ import { RTKResult } from "~/types/errors";
 export const DefaultCell = ({
   value,
 }: {
-  value: string | undefined | number | boolean;
+  value: string | undefined | number | null | boolean;
 }) => (
   <Flex alignItems="center" height="100%">
     <Text
@@ -60,7 +60,7 @@ const FidesBadge = ({ children, ...props }: BadgeProps) => (
 export const RelativeTimestampCell = ({
   time,
 }: {
-  time?: string | number | Date;
+  time?: string | number | Date | null;
 }) => {
   if (!time) {
     return <DefaultCell value="N/A" />;

clients/admin-ui/src/features/configure-consent/AddVendor.tsx

@@ -171,7 +171,7 @@ const AddVendor = ({
     handleCloseModal();
   };
 
-  const handleVendorSelected = (vendorId?: string) => {
+  const handleVendorSelected = (vendorId?: string | null) => {
     if (!dictionaryService) {
       return;
     }

clients/admin-ui/src/features/configure-consent/ConsentManagementTable.tsx

@@ -295,7 +295,7 @@ export const ConsentManagementTable = () => {
       </TableActionBar>
       <FidesTableV2 tableInstance={tableInstance} onRowClick={onRowClick} />
       <PaginationBar
-        totalRows={totalRows}
+        totalRows={totalRows || 0}
         pageSizes={PAGE_SIZES}
         setPageSize={setPageSize}
         onPreviousPageClick={onPreviousPageClick}

clients/admin-ui/src/features/consent-settings/GppConfiguration.tsx

@@ -76,14 +76,14 @@ const GppConfiguration = () => {
                   name="gpp.mspa_service_provider_mode"
                   variant="switchFirst"
                   tooltip="Enable service provider mode if you do not engage in any sales or sharing of personal information."
-                  isDisabled={values.gpp.mspa_opt_out_option_mode}
+                  isDisabled={Boolean(values.gpp.mspa_opt_out_option_mode)}
                 />
                 <CustomSwitch
                   label="Enable MSPA opt-out option mode"
                   name="gpp.mspa_opt_out_option_mode"
                   variant="switchFirst"
                   tooltip="Enable opt-out option mode if you engage or may engage in the sales or sharing of personal information, or process any information for the purpose of targeted advertising."
-                  isDisabled={values.gpp.mspa_service_provider_mode}
+                  isDisabled={Boolean(values.gpp.mspa_service_provider_mode)}
                 />
               </Section>
             ) : null}

clients/admin-ui/src/features/data-discovery-and-detection/tables/ActivityTable.tsx

@@ -197,7 +197,7 @@ const ActivityTable = ({
         emptyTableNotice={<EmptyTableNotice />}
       />
       <PaginationBar
-        totalRows={totalRows}
+        totalRows={totalRows || 0}
         pageSizes={PAGE_SIZES}
         setPageSize={setPageSize}
         onPreviousPageClick={onPreviousPageClick}

clients/admin-ui/src/features/data-discovery-and-detection/tables/DetectionResultTable.tsx

@@ -195,7 +195,7 @@ const DetectionResultTable = ({ resourceUrn }: MonitorResultTableProps) => {
         emptyTableNotice={<EmptyTableNotice />}
       />
       <PaginationBar
-        totalRows={totalRows}
+        totalRows={totalRows || 0}
         pageSizes={PAGE_SIZES}
         setPageSize={setPageSize}
         onPreviousPageClick={onPreviousPageClick}

clients/admin-ui/src/features/data-discovery-and-detection/tables/DiscoveryResultTable.tsx

@@ -179,7 +179,7 @@ const DiscoveryResultTable = ({ resourceUrn }: MonitorResultTableProps) => {
         emptyTableNotice={<EmptyTableNotice />}
       />
       <PaginationBar
-        totalRows={totalRows}
+        totalRows={totalRows || 0}
         pageSizes={PAGE_SIZES}
         setPageSize={setPageSize}
         onPreviousPageClick={onPreviousPageClick}

clients/admin-ui/src/features/data-discovery-and-detection/types/DiscoveryMonitorItem.ts

@@ -3,4 +3,9 @@ import { Field, StagedResourceAPIResponse } from "~/types/api";
 /**
  * Utility class for a staged resource of unknown type
  */
-export type DiscoveryMonitorItem = StagedResourceAPIResponse & Partial<Field>;
+export type DiscoveryMonitorItem = StagedResourceAPIResponse &
+  Omit<Partial<Field>, "schema_name" | "parent_table_urn" | "table_name"> & {
+    schema_name?: string | null;
+    parent_table_urn?: string | null;
+    table_name?: string | null;
+  };

clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx

@@ -1172,7 +1172,7 @@ export const DatamapReportTable = () => {
 
       <FidesTableV2<DatamapReport> tableInstance={tableInstance} />
       <PaginationBar
-        totalRows={totalRows}
+        totalRows={totalRows || 0}
         pageSizes={PAGE_SIZES}
         setPageSize={setPageSize}
         onPreviousPageClick={onPreviousPageClick}

clients/admin-ui/src/features/dataset/types.ts

@@ -1,5 +1,5 @@
 import { DataCategory } from "~/types/api";
 
 export interface DataCategoryWithConfidence extends DataCategory {
-  confidence?: number;
+  confidence?: number | null;
 }

clients/admin-ui/src/features/datastore-connections/TestData.tsx

@@ -4,7 +4,7 @@ import ConnectedCircle from "../common/ConnectedCircle";
 import { formatDate } from "../common/utils";
 
 type TestDataProps = {
-  succeeded?: boolean;
+  succeeded?: boolean | null;
   timestamp: string | number;
 };
 

clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionForm.tsx

@@ -27,7 +27,7 @@ export type ConnectionOption = {
 };
 
 type Props = {
-  connectionConfig?: ConnectionConfigurationResponse;
+  connectionConfig?: ConnectionConfigurationResponse | null;
   systemFidesKey: string;
 };
 
@@ -69,7 +69,7 @@ const ConnectionForm = ({ connectionConfig, systemFidesKey }: Props) => {
             label="Integration type"
             selectedValue={selectedConnectionOption}
             onChange={setSelectedConnectionOption}
-            disabled={connectionConfig && connectionConfig !== null}
+            disabled={Boolean(connectionConfig && connectionConfig !== null)}
           />
           {!connectionConfig && orphanedConnectionConfigs.length > 0 ? (
             <OrphanedConnectionModal

clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionListDropdown.tsx

@@ -76,7 +76,7 @@ type SelectDropdownProps = {
 };
 
 type UseConnectionListDropDown = {
-  connectionConfig?: ConnectionConfigurationResponse;
+  connectionConfig?: ConnectionConfigurationResponse | null;
 };
 
 export const useConnectionListDropDown = ({

clients/admin-ui/src/features/datastore-connections/system_portal_config/ConsentAutomationForm.tsx

@@ -30,7 +30,7 @@ import {
   selectPageSize as selectNoticePageSize,
   useGetAllPrivacyNoticesQuery,
 } from "~/features/privacy-notices/privacy-notices.slice";
-import { ConsentableItem } from "~/types/api/models/ConsentableItem";
+import { ConsentableItem } from "~/types/api";
 import { isErrorResult } from "~/types/errors";
 
 interface ConsentableItemFieldProps {

clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParameters.tsx

@@ -190,7 +190,7 @@ type ConnectorParametersProps = {
   setSelectedConnectionOption: (
     option: ConnectionSystemTypeMap | undefined,
   ) => void;
-  connectionConfig?: ConnectionConfigurationResponse;
+  connectionConfig?: ConnectionConfigurationResponse | null;
 };
 
 export const useConnectorForm = ({
@@ -241,7 +241,7 @@ export const useConnectorForm = ({
   const { plus: isPlusEnabled } = useFeatures();
 
   const originalSecrets = useMemo(
-    () => (connectionConfig ? { ...connectionConfig.secrets } : {}),
+    () => connectionConfig?.secrets ?? {},
     [connectionConfig],
   );
   const activeSystem = useAppSelector(selectActiveSystem) as SystemResponse;

clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx

@@ -71,7 +71,7 @@ type ConnectorParametersFormProps = {
    * Parent callback when Authorize Connection is clicked
    */
   onAuthorizeConnectionClick: (values: ConnectionConfigFormValues) => void;
-  connectionConfig?: ConnectionConfigurationResponse;
+  connectionConfig?: ConnectionConfigurationResponse | null;
   connectionOption: ConnectionSystemTypeMap;
   isCreatingConnectionConfig: boolean;
   datasetDropdownOptions: Option[];
@@ -236,7 +236,7 @@ export const ConnectorParametersForm = ({
       ).map((action) => action.toString());
 
       // @ts-ignore
-      initialValues.secrets = { ...connectionConfig.secrets };
+      initialValues.secrets = connectionConfig.secrets ?? {};
 
       // check if we need we need to pre-process any secrets values
       // we currently only need to do this for Fides dataset references

clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/DSRCustomizationForm/DSRCustomizationModal.tsx

@@ -31,7 +31,7 @@ import DSRCustomizationForm from "./DSRCustomizationForm";
 import { Field } from "./types";
 
 type Props = {
-  connectionConfig?: ConnectionConfigurationResponse;
+  connectionConfig?: ConnectionConfigurationResponse | null;
 };
 
 const DSRCustomizationModal = ({ connectionConfig }: Props) => {

clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/fields/DatasetConfigField/DatasetConfigField.tsx

@@ -116,7 +116,7 @@ export const DatasetSelect = ({
 };
 
 type UseDatasetConfigField = {
-  connectionConfig?: ConnectionConfigurationResponse;
+  connectionConfig?: ConnectionConfigurationResponse | null;
 };
 export const useDatasetConfigField = ({
   connectionConfig,
@@ -230,7 +230,7 @@ export const useDatasetConfigField = ({
 
 type Props = {
   dropdownOptions: Option[];
-  connectionConfig?: ConnectionConfigurationResponse;
+  connectionConfig?: ConnectionConfigurationResponse | null;
 };
 
 const DatasetConfigField = ({ dropdownOptions, connectionConfig }: Props) => {

clients/admin-ui/src/features/datastore-connections/useTestConnection.tsx

@@ -42,7 +42,7 @@ const useTestConnection = (
       : integration?.last_test_timestamp,
     succeeded: data
       ? data.test_status === "succeeded"
-      : integration?.last_test_succeeded,
+      : Boolean(integration?.last_test_succeeded),
   };
 
   const isLoading = queryIsLoading || isFetching;

clients/admin-ui/src/features/integrations/ConnectionStatusNotice.tsx

@@ -3,7 +3,7 @@ import { CheckCircleIcon, Flex, Text, WarningTwoIcon } from "fidesui";
 import { formatDate } from "~/features/common/utils";
 
 export type ConnectionStatusData = {
-  timestamp?: string;
+  timestamp?: string | null;
   succeeded?: boolean;
 };