diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml index dc961c00c8..211b03976e 100644 --- a/.github/workflows/backend_checks.yml +++ b/.github/workflows/backend_checks.yml @@ -43,7 +43,7 @@ jobs: strategy: matrix: # NOTE: These are the currently supported/tested Python Versions - python_version: ["3.8.18", "3.9.18", "3.10.13"] + python_version: ["3.9.18", "3.10.13"] runs-on: ubuntu-latest steps: - name: Checkout @@ -223,7 +223,7 @@ jobs: needs: Check-Container-Startup strategy: matrix: - python_version: ["3.8.18", "3.9.18", "3.10.13"] + python_version: ["3.9.18", "3.10.13"] test_selection: - "ctl-not-external" - "ops-unit-api" @@ -275,7 +275,7 @@ jobs: strategy: max-parallel: 1 # This prevents collisions in shared external resources matrix: - python_version: ["3.8.18", "3.9.18", "3.10.13"] + python_version: ["3.9.18", "3.10.13"] runs-on: ubuntu-latest timeout-minutes: 20 # In PRs run with the "unsafe" label, or run on a "push" event to main @@ -315,7 +315,7 @@ jobs: strategy: max-parallel: 1 # This prevents collisions in shared external resources matrix: - python_version: ["3.8.18", "3.9.18", "3.10.13"] + python_version: ["3.9.18", "3.10.13"] runs-on: ubuntu-latest timeout-minutes: 20 # In PRs run with the "unsafe" label, or run on a "push" event to main @@ -381,7 +381,7 @@ jobs: strategy: max-parallel: 1 # This prevents collisions in shared external resources matrix: - python_version: ["3.8.18", "3.9.18", "3.10.13"] + python_version: ["3.9.18", "3.10.13"] steps: - name: Download container uses: actions/download-artifact@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6311552560..59bc481601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,18 @@ The types of changes are: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. -## [Unreleased](https://github.com/ethyca/fides/compare/2.43.0...main) +## [Unreleased](https://github.com/ethyca/fides/compare/2.43.1...main) + + + +## [2.43.1](https://github.com/ethyca/fides/compare/2.43.0...2.43.1) + +### Added +- Pydantic v1 -> Pydantic v2 upgrade [#5020](https://github.com/ethyca/fides/pull/5020) + +### Fixed +- Ignore `404` errors on Oracle Responsys deletions [#5203](https://github.com/ethyca/fides/pull/5203) +- Fix white screen issue when privacy request has null value for daysLeft [#5213](https://github.com/ethyca/fides/pull/5213) ## [2.43.0](https://github.com/ethyca/fides/compare/2.42.1...2.43.0) diff --git a/README.md b/README.md index 1b0a769f69..465497534b 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ In order to get started quickly with Fides, a sample project is bundled within t #### Minimum requirements (for all platforms) * [Docker](https://www.docker.com/products/docker-desktop) (version 20.10.11 or later) -* [Python](https://www.python.org/downloads/) (version 3.8 through 3.10) +* [Python](https://www.python.org/downloads/) (version 3.9 through 3.10) #### Download and install Fides diff --git a/clients/admin-ui/src/features/common/ConnectedCircle.tsx b/clients/admin-ui/src/features/common/ConnectedCircle.tsx index 44027a933e..de332ca04c 100644 --- a/clients/admin-ui/src/features/common/ConnectedCircle.tsx +++ b/clients/admin-ui/src/features/common/ConnectedCircle.tsx @@ -3,7 +3,7 @@ import { Box, BoxProps } from "fidesui"; const ConnectedCircle = ({ connected, ...other -}: { connected?: boolean } & BoxProps) => { +}: { connected?: boolean | null } & BoxProps) => { let color = "red.500"; if (connected == null) { color = "gray.300"; diff --git a/clients/admin-ui/src/features/common/SystemsCheckboxTable.tsx b/clients/admin-ui/src/features/common/SystemsCheckboxTable.tsx index 778e88ed86..3ac1d58b40 100644 --- a/clients/admin-ui/src/features/common/SystemsCheckboxTable.tsx +++ b/clients/admin-ui/src/features/common/SystemsCheckboxTable.tsx @@ -54,7 +54,7 @@ export const SystemTableCell = ({ whiteSpace="nowrap" overflow="hidden" textOverflow="ellipsis" - title={system.fidesctl_meta?.resource_id} + title={system.fidesctl_meta?.resource_id || ""} > {system.fidesctl_meta?.resource_id} diff --git a/clients/admin-ui/src/features/common/form/inputs.tsx b/clients/admin-ui/src/features/common/form/inputs.tsx index 7a25c93951..67a4ca03e4 100644 --- a/clients/admin-ui/src/features/common/form/inputs.tsx +++ b/clients/admin-ui/src/features/common/form/inputs.tsx @@ -64,7 +64,7 @@ type Variant = "inline" | "stacked" | "block"; export interface CustomInputProps { disabled?: boolean; label?: string; - tooltip?: string; + tooltip?: string | null; variant?: Variant; isRequired?: boolean; textColor?: string; @@ -172,7 +172,7 @@ const ClearIndicator = () => null; export interface Option { value: string; label: string; - description?: string; + description?: string | null; tooltip?: string; } @@ -204,8 +204,8 @@ export interface SelectProps { label?: string; labelProps?: FormLabelProps; placeholder?: string; - tooltip?: string; - options: Option[]; + tooltip?: string | null; + options?: Option[] | []; isDisabled?: boolean; isSearchable?: boolean; isClearable?: boolean; @@ -247,7 +247,7 @@ export const SELECT_STYLES: ChakraStylesConfig< }; export const SelectInput = ({ - options, + options = [], fieldName, placeholder, size, @@ -399,7 +399,7 @@ interface CreatableSelectProps extends SelectProps { disableMenu?: boolean; } const CreatableSelectInput = ({ - options, + options = [], placeholder, fieldName, size, diff --git a/clients/admin-ui/src/features/common/hooks/usePicker.ts b/clients/admin-ui/src/features/common/hooks/usePicker.ts index 8be358d87c..e27236d060 100644 --- a/clients/admin-ui/src/features/common/hooks/usePicker.ts +++ b/clients/admin-ui/src/features/common/hooks/usePicker.ts @@ -51,7 +51,7 @@ interface UsePaginatedPickerProps { initialSelected: string[]; initialExcluded: string[]; initialAllSelected?: boolean; - itemCount: number; + itemCount: number | null; } export const usePaginatedPicker = ({ diff --git a/clients/admin-ui/src/features/common/hooks/useTaxonomies.tsx b/clients/admin-ui/src/features/common/hooks/useTaxonomies.tsx index ec6f48feaf..c39e5da469 100644 --- a/clients/admin-ui/src/features/common/hooks/useTaxonomies.tsx +++ b/clients/admin-ui/src/features/common/hooks/useTaxonomies.tsx @@ -35,8 +35,8 @@ const useTaxonomies = () => { fidesLangKey: string, getDataFunction: (fidesLangKey: string) => | { - parent_key?: string; - name?: string; + parent_key?: string | null; + name?: string | null; } | undefined, primaryLevel = 1, diff --git a/clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordion.tsx b/clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordion.tsx index 4766072d02..19d7997f6c 100644 --- a/clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordion.tsx +++ b/clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordion.tsx @@ -1,7 +1,7 @@ import { Accordion } from "fidesui"; import React from "react"; -import { System } from "~/types/api/models/System"; +import { System } from "~/types/api"; import { DataFlowAccordionForm } from "./DataFlowAccordionForm"; diff --git a/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx b/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx index cc554ebf52..764d41aa06 100644 --- a/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx +++ b/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx @@ -45,7 +45,7 @@ export const useServerSidePagination = () => { const defaultPageIndex = 1; const [pageSize, setPageSize] = useState(PAGE_SIZES[0]); const [pageIndex, setPageIndex] = useState(defaultPageIndex); - const [totalPages, setTotalPages] = useState(); + const [totalPages, setTotalPages] = useState(); const onPreviousPageClick = useCallback(() => { setPageIndex((prev) => prev - 1); }, [setPageIndex]); diff --git a/clients/admin-ui/src/features/common/table/v2/cells.tsx b/clients/admin-ui/src/features/common/table/v2/cells.tsx index 9e03ffd75e..9f31db57e9 100644 --- a/clients/admin-ui/src/features/common/table/v2/cells.tsx +++ b/clients/admin-ui/src/features/common/table/v2/cells.tsx @@ -27,7 +27,7 @@ import { RTKResult } from "~/types/errors"; export const DefaultCell = ({ value, }: { - value: string | undefined | number | boolean; + value: string | undefined | number | null | boolean; }) => ( ( export const RelativeTimestampCell = ({ time, }: { - time?: string | number | Date; + time?: string | number | Date | null; }) => { if (!time) { return ; diff --git a/clients/admin-ui/src/features/configure-consent/AddVendor.tsx b/clients/admin-ui/src/features/configure-consent/AddVendor.tsx index f4ee1b3a88..14672e4659 100644 --- a/clients/admin-ui/src/features/configure-consent/AddVendor.tsx +++ b/clients/admin-ui/src/features/configure-consent/AddVendor.tsx @@ -171,7 +171,7 @@ const AddVendor = ({ handleCloseModal(); }; - const handleVendorSelected = (vendorId?: string) => { + const handleVendorSelected = (vendorId?: string | null) => { if (!dictionaryService) { return; } diff --git a/clients/admin-ui/src/features/configure-consent/ConsentManagementTable.tsx b/clients/admin-ui/src/features/configure-consent/ConsentManagementTable.tsx index f05accde58..ddcb28371e 100644 --- a/clients/admin-ui/src/features/configure-consent/ConsentManagementTable.tsx +++ b/clients/admin-ui/src/features/configure-consent/ConsentManagementTable.tsx @@ -295,7 +295,7 @@ export const ConsentManagementTable = () => { { name="gpp.mspa_service_provider_mode" variant="switchFirst" tooltip="Enable service provider mode if you do not engage in any sales or sharing of personal information." - isDisabled={values.gpp.mspa_opt_out_option_mode} + isDisabled={Boolean(values.gpp.mspa_opt_out_option_mode)} /> ) : null} diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/tables/ActivityTable.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/tables/ActivityTable.tsx index 3d57acd502..a7660fac06 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/tables/ActivityTable.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/tables/ActivityTable.tsx @@ -197,7 +197,7 @@ const ActivityTable = ({ emptyTableNotice={} /> { emptyTableNotice={} /> { emptyTableNotice={} /> ; +export type DiscoveryMonitorItem = StagedResourceAPIResponse & + Omit, "schema_name" | "parent_table_urn" | "table_name"> & { + schema_name?: string | null; + parent_table_urn?: string | null; + table_name?: string | null; + }; diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx index a1fd5399fb..3d24eee0a0 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx @@ -1172,7 +1172,7 @@ export const DatamapReportTable = () => { tableInstance={tableInstance} /> { label="Integration type" selectedValue={selectedConnectionOption} onChange={setSelectedConnectionOption} - disabled={connectionConfig && connectionConfig !== null} + disabled={Boolean(connectionConfig && connectionConfig !== null)} /> {!connectionConfig && orphanedConnectionConfigs.length > 0 ? ( void; - connectionConfig?: ConnectionConfigurationResponse; + connectionConfig?: ConnectionConfigurationResponse | null; }; export const useConnectorForm = ({ @@ -241,7 +241,7 @@ export const useConnectorForm = ({ const { plus: isPlusEnabled } = useFeatures(); const originalSecrets = useMemo( - () => (connectionConfig ? { ...connectionConfig.secrets } : {}), + () => connectionConfig?.secrets ?? {}, [connectionConfig], ); const activeSystem = useAppSelector(selectActiveSystem) as SystemResponse; diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx index 7eaa26e489..f471306391 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx @@ -71,7 +71,7 @@ type ConnectorParametersFormProps = { * Parent callback when Authorize Connection is clicked */ onAuthorizeConnectionClick: (values: ConnectionConfigFormValues) => void; - connectionConfig?: ConnectionConfigurationResponse; + connectionConfig?: ConnectionConfigurationResponse | null; connectionOption: ConnectionSystemTypeMap; isCreatingConnectionConfig: boolean; datasetDropdownOptions: Option[]; @@ -236,7 +236,7 @@ export const ConnectorParametersForm = ({ ).map((action) => action.toString()); // @ts-ignore - initialValues.secrets = { ...connectionConfig.secrets }; + initialValues.secrets = connectionConfig.secrets ? _.cloneDeep(connectionConfig.secrets) : {}; // check if we need we need to pre-process any secrets values // we currently only need to do this for Fides dataset references @@ -245,8 +245,10 @@ export const ConnectorParametersForm = ({ Object.entries(secretsSchema.properties).forEach(([key, schema]) => { if (schema.allOf?.[0].$ref === FIDES_DATASET_REFERENCE) { const datasetReference = initialValues.secrets[key]; - initialValues.secrets[key] = - `${datasetReference.dataset}.${datasetReference.field}`; + if (datasetReference) { + initialValues.secrets[key] = + `${datasetReference.dataset}.${datasetReference.field}`; + } } }); } diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/DSRCustomizationForm/DSRCustomizationModal.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/DSRCustomizationForm/DSRCustomizationModal.tsx index 03b3363112..241d1f8c00 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/DSRCustomizationForm/DSRCustomizationModal.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/DSRCustomizationForm/DSRCustomizationModal.tsx @@ -31,7 +31,7 @@ import DSRCustomizationForm from "./DSRCustomizationForm"; import { Field } from "./types"; type Props = { - connectionConfig?: ConnectionConfigurationResponse; + connectionConfig?: ConnectionConfigurationResponse | null; }; const DSRCustomizationModal = ({ connectionConfig }: Props) => { diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/fields/DatasetConfigField/DatasetConfigField.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/fields/DatasetConfigField/DatasetConfigField.tsx index 9ca7095966..4cc616bf58 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/fields/DatasetConfigField/DatasetConfigField.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/fields/DatasetConfigField/DatasetConfigField.tsx @@ -116,7 +116,7 @@ export const DatasetSelect = ({ }; type UseDatasetConfigField = { - connectionConfig?: ConnectionConfigurationResponse; + connectionConfig?: ConnectionConfigurationResponse | null; }; export const useDatasetConfigField = ({ connectionConfig, @@ -230,7 +230,7 @@ export const useDatasetConfigField = ({ type Props = { dropdownOptions: Option[]; - connectionConfig?: ConnectionConfigurationResponse; + connectionConfig?: ConnectionConfigurationResponse | null; }; const DatasetConfigField = ({ dropdownOptions, connectionConfig }: Props) => { diff --git a/clients/admin-ui/src/features/datastore-connections/useTestConnection.tsx b/clients/admin-ui/src/features/datastore-connections/useTestConnection.tsx index 0848a86810..d234261c1e 100644 --- a/clients/admin-ui/src/features/datastore-connections/useTestConnection.tsx +++ b/clients/admin-ui/src/features/datastore-connections/useTestConnection.tsx @@ -42,7 +42,7 @@ const useTestConnection = ( : integration?.last_test_timestamp, succeeded: data ? data.test_status === "succeeded" - : integration?.last_test_succeeded, + : Boolean(integration?.last_test_succeeded), }; const isLoading = queryIsLoading || isFetching; diff --git a/clients/admin-ui/src/features/integrations/ConnectionStatusNotice.tsx b/clients/admin-ui/src/features/integrations/ConnectionStatusNotice.tsx index 0efbf76add..3c5ab4114b 100644 --- a/clients/admin-ui/src/features/integrations/ConnectionStatusNotice.tsx +++ b/clients/admin-ui/src/features/integrations/ConnectionStatusNotice.tsx @@ -3,7 +3,7 @@ import { CheckCircleIcon, Flex, Text, WarningTwoIcon } from "fidesui"; import { formatDate } from "~/features/common/utils"; export type ConnectionStatusData = { - timestamp?: string; + timestamp?: string | null; succeeded?: boolean; }; diff --git a/clients/admin-ui/src/features/integrations/configure-monitor/MonitorConfigTab.tsx b/clients/admin-ui/src/features/integrations/configure-monitor/MonitorConfigTab.tsx index e028f5220e..387f402689 100644 --- a/clients/admin-ui/src/features/integrations/configure-monitor/MonitorConfigTab.tsx +++ b/clients/admin-ui/src/features/integrations/configure-monitor/MonitorConfigTab.tsx @@ -246,7 +246,7 @@ const MonitorConfigTab = ({ emptyTableNotice={} /> { if (dataCategories?.length) { diff --git a/clients/admin-ui/src/features/privacy-experience/PrivacyExperienceTranslationForm.tsx b/clients/admin-ui/src/features/privacy-experience/PrivacyExperienceTranslationForm.tsx index fb85023595..632e631a86 100644 --- a/clients/admin-ui/src/features/privacy-experience/PrivacyExperienceTranslationForm.tsx +++ b/clients/admin-ui/src/features/privacy-experience/PrivacyExperienceTranslationForm.tsx @@ -182,7 +182,7 @@ const PrivacyExperienceTranslationForm = ({ name={`translations.${translationIndex}.is_default`} id={`translations.${translationIndex}.is_default`} label="Default language" - isDisabled={initialTranslation.is_default} + isDisabled={Boolean(initialTranslation.is_default)} variant="stacked" /> { emptyTableNotice={} /> { emptyTableNotice={} /> { onSort={handleSort} /> { if (!identityInputs) { return null; @@ -43,7 +43,7 @@ const IdentityFields = ({ ) : null} @@ -51,7 +51,7 @@ const IdentityFields = ({ ) : null} @@ -65,7 +65,7 @@ const CustomFields = ({ customFieldInputs?: Record< string, fides__api__schemas__privacy_center_config__CustomPrivacyRequestField - >; + > | null; }) => { if (!customFieldInputs) { return null; @@ -78,7 +78,7 @@ const CustomFields = ({ name={`custom_privacy_request_fields.${fieldName}.value`} key={fieldName} label={fieldInfo.label} - isRequired={fieldInfo.required} + isRequired={Boolean(fieldInfo.required)} variant="stacked" /> ))} diff --git a/clients/admin-ui/src/features/privacy-requests/cells.tsx b/clients/admin-ui/src/features/privacy-requests/cells.tsx index 49b0e13b96..c06fc79f2e 100644 --- a/clients/admin-ui/src/features/privacy-requests/cells.tsx +++ b/clients/admin-ui/src/features/privacy-requests/cells.tsx @@ -75,6 +75,7 @@ export const RequestDaysLeftCell = ({ }) => { if ( daysLeft === undefined || + daysLeft === null || status === PrivacyRequestStatus.COMPLETE || status === PrivacyRequestStatus.CANCELED || status === PrivacyRequestStatus.DENIED || diff --git a/clients/admin-ui/src/features/properties/PropertiesTable.tsx b/clients/admin-ui/src/features/properties/PropertiesTable.tsx index 61a23660e1..3e9f21982c 100644 --- a/clients/admin-ui/src/features/properties/PropertiesTable.tsx +++ b/clients/admin-ui/src/features/properties/PropertiesTable.tsx @@ -197,7 +197,7 @@ export const PropertiesTable = () => { emptyTableNotice={} /> ; + messaging_templates?: Array | null; experiences: Array; } diff --git a/clients/admin-ui/src/features/system/SystemInformationForm.tsx b/clients/admin-ui/src/features/system/SystemInformationForm.tsx index 28b917b815..7c9d39dfef 100644 --- a/clients/admin-ui/src/features/system/SystemInformationForm.tsx +++ b/clients/admin-ui/src/features/system/SystemInformationForm.tsx @@ -257,7 +257,7 @@ const SystemInformationForm = ({ handleResult(result); }; - const handleVendorSelected = (newVendorId: string | undefined) => { + const handleVendorSelected = (newVendorId?: string | null) => { if (!features.dictionaryService) { return; } diff --git a/clients/admin-ui/src/features/system/VendorSelector.tsx b/clients/admin-ui/src/features/system/VendorSelector.tsx index 2514f92997..cbdf31b84a 100644 --- a/clients/admin-ui/src/features/system/VendorSelector.tsx +++ b/clients/admin-ui/src/features/system/VendorSelector.tsx @@ -82,7 +82,7 @@ interface Props { isCreate: boolean; lockedForGVL: boolean; options: DictOption[]; - onVendorSelected: (vendorId: string | undefined) => void; + onVendorSelected: (vendorId?: string | null) => void; } const CustomDictOption = ({ diff --git a/clients/admin-ui/src/features/system/add-multiple-systems/AddMultipleSystems.tsx b/clients/admin-ui/src/features/system/add-multiple-systems/AddMultipleSystems.tsx index 4da41acdfc..5a764f5932 100644 --- a/clients/admin-ui/src/features/system/add-multiple-systems/AddMultipleSystems.tsx +++ b/clients/admin-ui/src/features/system/add-multiple-systems/AddMultipleSystems.tsx @@ -420,7 +420,7 @@ export const AddMultipleSystems = ({ redirectRoute }: Props) => { /> { const { formattedTime, formattedDate } = formatDateAndTime(system.created_at); - const totalPages = data ? Math.ceil(data.total / ITEMS_PER_PAGE) : 0; + const totalPages = + data && data.total ? Math.ceil(data.total / ITEMS_PER_PAGE) : 0; const handleNextPage = () => { if (currentPage < totalPages) { diff --git a/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationDisplayGroup.tsx b/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationDisplayGroup.tsx index 5c087e2103..f884dec7de 100644 --- a/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationDisplayGroup.tsx +++ b/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationDisplayGroup.tsx @@ -27,7 +27,7 @@ const PrivacyDeclarationRow = ({ handleEdit, }: { declaration: PrivacyDeclarationResponse; - title?: string; + title?: string | null; handleDelete?: (dec: PrivacyDeclarationResponse) => void; handleEdit: (dec: PrivacyDeclarationResponse) => void; }) => { diff --git a/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationForm.tsx b/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationForm.tsx index f06fe7e3ed..bf79f54924 100644 --- a/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationForm.tsx +++ b/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationForm.tsx @@ -98,7 +98,7 @@ export interface DataProps { allDataSubjects: DataSubject[]; allDatasets?: Dataset[]; includeCustomFields?: boolean; - cookies?: Cookies[]; + cookies?: Cookies[] | null; } export const PrivacyDeclarationFormComponents = ({ diff --git a/clients/admin-ui/src/features/taxonomy/types.ts b/clients/admin-ui/src/features/taxonomy/types.ts index 60d0be5e71..1cf4a180ed 100644 --- a/clients/admin-ui/src/features/taxonomy/types.ts +++ b/clients/admin-ui/src/features/taxonomy/types.ts @@ -2,7 +2,7 @@ import { CustomFieldsFormValues } from "~/features/common/custom-fields"; import { TreeNode } from "~/features/common/types"; export interface TaxonomyEntityNode extends TreeNode { - description?: string; + description?: string | null; children: TaxonomyEntityNode[]; is_default: boolean; active: boolean; @@ -10,14 +10,14 @@ export interface TaxonomyEntityNode extends TreeNode { export interface TaxonomyEntity { fides_key: string; - name?: string; - description?: string; - parent_key?: string; + name?: string | null; + description?: string | null; + parent_key?: string | null; is_default?: boolean; active?: boolean; - version_added?: string; - version_deprecated?: string; - replaced_by?: string; + version_added?: string | null; + version_deprecated?: string | null; + replaced_by?: string | null; } export interface Labels { diff --git a/clients/admin-ui/src/features/user-management/UserManagementTable.tsx b/clients/admin-ui/src/features/user-management/UserManagementTable.tsx index 7ddfe9c431..420c8138f4 100644 --- a/clients/admin-ui/src/features/user-management/UserManagementTable.tsx +++ b/clients/admin-ui/src/features/user-management/UserManagementTable.tsx @@ -25,7 +25,8 @@ const useUsersTable = () => { }; const { data, isLoading } = useGetAllUsersQuery(filters); - const { items: users, total } = data || { users: [], total: 0 }; + const users = data?.items ?? []; + const total = data?.total ?? 0; return { ...filters, diff --git a/clients/admin-ui/src/pages/dataset/index.tsx b/clients/admin-ui/src/pages/dataset/index.tsx index 7affb91ebc..166d0bae19 100644 --- a/clients/admin-ui/src/pages/dataset/index.tsx +++ b/clients/admin-ui/src/pages/dataset/index.tsx @@ -230,7 +230,7 @@ const DataSets: NextPage = () => { )} { /> )} { )} ; - id?: string; + name?: string | null; + description?: string | null; + allowed_values?: Array | null; + id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/AllowListUpdate.ts b/clients/admin-ui/src/types/api/models/AllowListUpdate.ts index 1f65532b59..e96b5f759e 100644 --- a/clients/admin-ui/src/types/api/models/AllowListUpdate.ts +++ b/clients/admin-ui/src/types/api/models/AllowListUpdate.ts @@ -4,7 +4,7 @@ export type AllowListUpdate = { name: string; - description?: string; + description?: string | null; allowed_values: Array; - id?: string; + id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/BasicSystemResponse.ts b/clients/admin-ui/src/types/api/models/BasicSystemResponse.ts index 0cc19c37da..1e70367468 100644 --- a/clients/admin-ui/src/types/api/models/BasicSystemResponse.ts +++ b/clients/admin-ui/src/types/api/models/BasicSystemResponse.ts @@ -25,19 +25,19 @@ export type BasicSystemResponse = { * Defines the Organization that this resource belongs to. */ organization_fides_key?: string; - tags?: Array; + tags?: Array | null; /** * Human-Readable name for this resource. */ - name?: string; + name?: string | null; /** * A detailed description of what this resource is. */ - description?: string; + description?: string | null; /** * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. */ - meta?: any; + meta?: null; /** * * The SystemMetadata resource model. @@ -45,7 +45,7 @@ export type BasicSystemResponse = { * Object used to hold application specific metadata for a system * */ - fidesctl_meta?: SystemMetadata; + fidesctl_meta?: SystemMetadata | null; /** * A required value to describe the type of system being modeled, examples include: Service, Application, Third Party, etc. */ @@ -53,11 +53,11 @@ export type BasicSystemResponse = { /** * The resources to which the system sends data. */ - egress?: Array; + egress?: Array | null; /** * The resources from which the system receives data. */ - ingress?: Array; + ingress?: Array | null; /** * * The PrivacyDeclaration resource model. @@ -70,19 +70,19 @@ export type BasicSystemResponse = { /** * An optional value to identify the owning department or group of the system within your organization */ - administrating_department?: string; + administrating_department?: string | null; /** * The unique identifier for the vendor that's associated with this system. */ - vendor_id?: string; + vendor_id?: string | null; /** * If specified, the unique identifier for the vendor that was previously associated with this system. */ - previous_vendor_id?: string; + previous_vendor_id?: string | null; /** * The deleted date of the vendor that's associated with this system. */ - vendor_deleted_date?: string; + vendor_deleted_date?: string | null; /** * Referenced Dataset fides keys used by the system. */ @@ -98,7 +98,7 @@ export type BasicSystemResponse = { /** * The reason that the system is exempt from privacy regulation. */ - reason_for_exemption?: string; + reason_for_exemption?: string | null; /** * Whether the vendor uses data to profile a consumer in a way that has a legal effect. */ @@ -122,23 +122,23 @@ export type BasicSystemResponse = { /** * Location where the DPAs or DIPAs can be found. */ - dpa_location?: string; + dpa_location?: string | null; /** * The optional status of a Data Protection Impact Assessment */ - dpa_progress?: string; + dpa_progress?: string | null; /** * A URL that points to the system's publicly accessible privacy policy. */ - privacy_policy?: string; + privacy_policy?: string | null; /** * The legal name for the business represented by the system. */ - legal_name?: string; + legal_name?: string | null; /** * The legal address for the business represented by the system. */ - legal_address?: string; + legal_address?: string | null; /** * * The model defining the responsibility or role over @@ -152,19 +152,19 @@ export type BasicSystemResponse = { /** * The official privacy contact address or DPO. */ - dpo?: string; + dpo?: string | null; /** * The party or parties that share the responsibility for processing personal data. */ - joint_controller_info?: string; + joint_controller_info?: string | null; /** * The data security practices employed by this system. */ - data_security_practices?: string; + data_security_practices?: string | null; /** * The maximum storage duration, in seconds, for cookies used by this system. */ - cookie_max_age_seconds?: number; + cookie_max_age_seconds?: number | null; /** * Whether this system uses cookie storage. */ @@ -180,10 +180,10 @@ export type BasicSystemResponse = { /** * A URL that points to the system's publicly accessible legitimate interest disclosure. */ - legitimate_interest_disclosure_url?: string; + legitimate_interest_disclosure_url?: string | null; /** * System-level cookies unassociated with a data use to deliver services and functionality */ - cookies?: Array; + cookies?: Array | null; created_at: string; }; diff --git a/clients/admin-ui/src/types/api/models/BigQueryConfig.ts b/clients/admin-ui/src/types/api/models/BigQueryConfig.ts index b38eb9f171..c3016a1725 100644 --- a/clients/admin-ui/src/types/api/models/BigQueryConfig.ts +++ b/clients/admin-ui/src/types/api/models/BigQueryConfig.ts @@ -8,6 +8,6 @@ import type { fides__connectors__models__KeyfileCreds } from "./fides__connector * The model for the connection config for BigQuery */ export type BigQueryConfig = { - dataset?: string; + dataset?: string | null; keyfile_creds: fides__connectors__models__KeyfileCreds; }; diff --git a/clients/admin-ui/src/types/api/models/BigQueryDocsSchema.ts b/clients/admin-ui/src/types/api/models/BigQueryDocsSchema.ts index 6a6be08cba..9a4c2b459b 100644 --- a/clients/admin-ui/src/types/api/models/BigQueryDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/BigQueryDocsSchema.ts @@ -15,5 +15,5 @@ export type BigQueryDocsSchema = { /** * The dataset within your BigQuery project that contains the tables you want to access. */ - dataset?: string; + dataset?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/Body_acquire_access_token_api_v1_oauth_token_post.ts b/clients/admin-ui/src/types/api/models/Body_acquire_access_token_api_v1_oauth_token_post.ts index fce6528a3d..8e03bb71ad 100644 --- a/clients/admin-ui/src/types/api/models/Body_acquire_access_token_api_v1_oauth_token_post.ts +++ b/clients/admin-ui/src/types/api/models/Body_acquire_access_token_api_v1_oauth_token_post.ts @@ -5,6 +5,6 @@ export type Body_acquire_access_token_api_v1_oauth_token_post = { grant_type?: string; scope?: string; - client_id?: string; - client_secret?: string; + client_id?: string | null; + client_secret?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/BulkCustomFieldRequest.ts b/clients/admin-ui/src/types/api/models/BulkCustomFieldRequest.ts index b797f72a99..aa2dc2b23d 100644 --- a/clients/admin-ui/src/types/api/models/BulkCustomFieldRequest.ts +++ b/clients/admin-ui/src/types/api/models/BulkCustomFieldRequest.ts @@ -8,6 +8,6 @@ import type { ResourceTypes } from "./ResourceTypes"; export type BulkCustomFieldRequest = { resource_type: ResourceTypes; resource_id: string; - upsert?: Array; - delete?: Array; + upsert?: Array | null; + delete?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/BulkPutBasicMessagingTemplateResponse.ts b/clients/admin-ui/src/types/api/models/BulkPutBasicMessagingTemplateResponse.ts index fc55641447..36f275e0b3 100644 --- a/clients/admin-ui/src/types/api/models/BulkPutBasicMessagingTemplateResponse.ts +++ b/clients/admin-ui/src/types/api/models/BulkPutBasicMessagingTemplateResponse.ts @@ -5,16 +5,6 @@ import type { BasicMessagingTemplateResponse } from "./BasicMessagingTemplateResponse"; import type { BulkUpdateFailed } from "./BulkUpdateFailed"; -/** - * Schema for responses from bulk update/create requests. Override to set "succeeded" and "failed" attributes on - * your child class with paginated types. - * - * Example: - * from fastapi_pagination import Page - * - * succeeded: List[ConnectionConfigurationResponse] - * failed: List[BulkUpdateFailed] - */ export type BulkPutBasicMessagingTemplateResponse = { succeeded: Array; failed: Array; diff --git a/clients/admin-ui/src/types/api/models/BulkPutStorageConfigResponse.ts b/clients/admin-ui/src/types/api/models/BulkPutStorageConfigResponse.ts index 951a1780b2..ae4ce31d8c 100644 --- a/clients/admin-ui/src/types/api/models/BulkPutStorageConfigResponse.ts +++ b/clients/admin-ui/src/types/api/models/BulkPutStorageConfigResponse.ts @@ -9,6 +9,6 @@ import type { StorageDestinationResponse } from "./StorageDestinationResponse"; * Schema with mixed success/failure responses for Bulk Create/Update of StorageConfig. */ export type BulkPutStorageConfigResponse = { - succeeded: Array; - failed: Array; + succeeded?: Array; + failed?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/CheckpointActionRequiredDetails.ts b/clients/admin-ui/src/types/api/models/CheckpointActionRequiredDetails.ts index 5b21ca3320..475db28e04 100644 --- a/clients/admin-ui/src/types/api/models/CheckpointActionRequiredDetails.ts +++ b/clients/admin-ui/src/types/api/models/CheckpointActionRequiredDetails.ts @@ -5,15 +5,8 @@ import type { CurrentStep } from "./CurrentStep"; import type { ManualAction } from "./ManualAction"; -/** - * Describes actions needed on a particular checkpoint. - * - * Examples are a paused collection that needs manual input, a failed collection that - * needs to be restarted, or a collection where instructions need to be emailed to a third - * party to complete the request. - */ export type CheckpointActionRequiredDetails = { step: CurrentStep; - collection?: string; - action_needed?: Array; + collection?: string | null; + action_needed?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/Classification.ts b/clients/admin-ui/src/types/api/models/Classification.ts index 1d0f4dd429..8fb28fbd04 100644 --- a/clients/admin-ui/src/types/api/models/Classification.ts +++ b/clients/admin-ui/src/types/api/models/Classification.ts @@ -8,6 +8,6 @@ export type Classification = { label: string; score: number; - aggregated_score?: number; - classification_paradigm?: string; + aggregated_score?: number | null; + classification_paradigm?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/ClassifyParams.ts b/clients/admin-ui/src/types/api/models/ClassifyParams.ts index a5dba54765..8e24860b5c 100644 --- a/clients/admin-ui/src/types/api/models/ClassifyParams.ts +++ b/clients/admin-ui/src/types/api/models/ClassifyParams.ts @@ -8,7 +8,7 @@ * and can rely on the defaults */ export type ClassifyParams = { - possible_targets?: Array; + possible_targets?: null; top_n?: number; remove_stop_words?: boolean; pii_threshold?: number; diff --git a/clients/admin-ui/src/types/api/models/ClientConfig.ts b/clients/admin-ui/src/types/api/models/ClientConfig.ts index 4c253e0b8c..c30b73a5eb 100644 --- a/clients/admin-ui/src/types/api/models/ClientConfig.ts +++ b/clients/admin-ui/src/types/api/models/ClientConfig.ts @@ -10,5 +10,5 @@ import type { Strategy } from "./Strategy"; export type ClientConfig = { protocol: string; host: string; - authentication?: Strategy; + authentication?: Strategy | null; }; diff --git a/clients/admin-ui/src/types/api/models/CloudConfig.ts b/clients/admin-ui/src/types/api/models/CloudConfig.ts index c6f8e75c93..162a8c5dad 100644 --- a/clients/admin-ui/src/types/api/models/CloudConfig.ts +++ b/clients/admin-ui/src/types/api/models/CloudConfig.ts @@ -3,6 +3,6 @@ /* eslint-disable */ export type CloudConfig = { - privacy_center_url?: string; + privacy_center_url?: string | null; domain_verification_records?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/CollectionAddressResponse.ts b/clients/admin-ui/src/types/api/models/CollectionAddressResponse.ts index 48d19ada65..b7c8847b0c 100644 --- a/clients/admin-ui/src/types/api/models/CollectionAddressResponse.ts +++ b/clients/admin-ui/src/types/api/models/CollectionAddressResponse.ts @@ -6,6 +6,6 @@ * Schema for the representation of a collection in the graph */ export type CollectionAddressResponse = { - dataset?: string; - collection?: string; + dataset: string | null; + collection: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/CollectionMeta.ts b/clients/admin-ui/src/types/api/models/CollectionMeta.ts index 8e5cf8ff81..3b6c8b670f 100644 --- a/clients/admin-ui/src/types/api/models/CollectionMeta.ts +++ b/clients/admin-ui/src/types/api/models/CollectionMeta.ts @@ -6,7 +6,6 @@ * Collection-level specific annotations used for query traversal */ export type CollectionMeta = { - after?: Array; - skip_processing?: boolean; - data_type?: string; + after?: Array | null; + skip_processing?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/ColumnSort.ts b/clients/admin-ui/src/types/api/models/ColumnSort.ts index 2a588e6371..d1e227473b 100644 --- a/clients/admin-ui/src/types/api/models/ColumnSort.ts +++ b/clients/admin-ui/src/types/api/models/ColumnSort.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum ColumnSort { DESC = "desc", ASC = "asc", diff --git a/clients/admin-ui/src/types/api/models/ConditionalValue.ts b/clients/admin-ui/src/types/api/models/ConditionalValue.ts index 3cfbeeafb9..e23182f76c 100644 --- a/clients/admin-ui/src/types/api/models/ConditionalValue.ts +++ b/clients/admin-ui/src/types/api/models/ConditionalValue.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type ConditionalValue = { value: boolean; globalPrivacyControl: boolean; diff --git a/clients/admin-ui/src/types/api/models/ConfigConsentOption.ts b/clients/admin-ui/src/types/api/models/ConfigConsentOption.ts index 498287e96c..52cd0d7fa1 100644 --- a/clients/admin-ui/src/types/api/models/ConfigConsentOption.ts +++ b/clients/admin-ui/src/types/api/models/ConfigConsentOption.ts @@ -4,16 +4,13 @@ import type { ConditionalValue } from "./ConditionalValue"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type ConfigConsentOption = { cookieKeys?: Array; - default?: boolean | ConditionalValue; + default?: boolean | ConditionalValue | null; description: string; fidesDataUseKey: string; - highlight?: boolean; + highlight?: boolean | null; name: string; url: string; - executable?: boolean; + executable?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts b/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts index 9060c70d33..bc511df7fe 100644 --- a/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts +++ b/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts @@ -11,18 +11,18 @@ import type { SaaSConfigBase } from "./SaaSConfigBase"; * Describes the returned schema for a ConnectionConfiguration. */ export type ConnectionConfigurationResponse = { - name?: string; + name?: string | null; key: string; - description?: string; + description?: string | null; connection_type: ConnectionType; access: AccessLevel; created_at: string; - updated_at?: string; - disabled?: boolean; - last_test_timestamp?: string; - last_test_succeeded?: boolean; - saas_config?: SaaSConfigBase; + updated_at?: string | null; + disabled?: boolean | null; + last_test_timestamp?: string | null; + last_test_succeeded?: boolean | null; + saas_config?: SaaSConfigBase | null; secrets?: any; - authorized?: boolean; - enabled_actions?: Array; + authorized?: boolean | null; + enabled_actions?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/ConnectionSystemTypeMap.ts b/clients/admin-ui/src/types/api/models/ConnectionSystemTypeMap.ts index 2216b965e6..b3292866a9 100644 --- a/clients/admin-ui/src/types/api/models/ConnectionSystemTypeMap.ts +++ b/clients/admin-ui/src/types/api/models/ConnectionSystemTypeMap.ts @@ -13,8 +13,8 @@ export type ConnectionSystemTypeMap = { identifier: ConnectionType | string; type: SystemType; human_readable: string; - encoded_icon?: string; - authorization_required?: boolean; - user_guide?: string; + encoded_icon?: string | null; + authorization_required?: boolean | null; + user_guide?: string | null; supported_actions: Array; }; diff --git a/clients/admin-ui/src/types/api/models/ConnectorParam.ts b/clients/admin-ui/src/types/api/models/ConnectorParam.ts index 00e0eb43a4..3ff8aad632 100644 --- a/clients/admin-ui/src/types/api/models/ConnectorParam.ts +++ b/clients/admin-ui/src/types/api/models/ConnectorParam.ts @@ -7,10 +7,10 @@ */ export type ConnectorParam = { name: string; - label?: string; - options?: Array; - default_value?: string | Array; - multiselect?: boolean; - description?: string; - sensitive?: boolean; + label?: string | null; + options?: Array | null; + default_value?: string | Array | number | Array | null; + multiselect?: boolean | null; + description?: string | null; + sensitive?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/Consent.ts b/clients/admin-ui/src/types/api/models/Consent.ts index 68960b5029..3e78c72802 100644 --- a/clients/admin-ui/src/types/api/models/Consent.ts +++ b/clients/admin-ui/src/types/api/models/Consent.ts @@ -7,7 +7,7 @@ */ export type Consent = { data_use: string; - data_use_description?: string; + data_use_description?: string | null; opt_in: boolean; has_gpc_flag?: boolean; conflicts_with_gpc?: boolean; diff --git a/clients/admin-ui/src/types/api/models/ConsentConfigButton.ts b/clients/admin-ui/src/types/api/models/ConsentConfigButton.ts index f33c54e699..ef5857fb37 100644 --- a/clients/admin-ui/src/types/api/models/ConsentConfigButton.ts +++ b/clients/admin-ui/src/types/api/models/ConsentConfigButton.ts @@ -5,20 +5,17 @@ import type { fides__api__schemas__privacy_center_config__CustomPrivacyRequestField } from "./fides__api__schemas__privacy_center_config__CustomPrivacyRequestField"; import type { IdentityInputs } from "./IdentityInputs"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type ConsentConfigButton = { description: string; - description_subtext?: Array; - confirmButtonText?: string; - cancelButtonText?: string; + description_subtext?: Array | null; + confirmButtonText?: string | null; + cancelButtonText?: string | null; icon_path: string; identity_inputs: IdentityInputs; custom_privacy_request_fields?: Record< string, fides__api__schemas__privacy_center_config__CustomPrivacyRequestField - >; + > | null; title: string; - modalTitle?: string; + modalTitle?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/ConsentConfigPage.ts b/clients/admin-ui/src/types/api/models/ConsentConfigPage.ts index fdab0b147d..dd42a52c14 100644 --- a/clients/admin-ui/src/types/api/models/ConsentConfigPage.ts +++ b/clients/admin-ui/src/types/api/models/ConsentConfigPage.ts @@ -4,13 +4,10 @@ import type { ConfigConsentOption } from "./ConfigConsentOption"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type ConsentConfigPage = { consentOptions?: Array; description: string; - description_subtext?: Array; - policy_key?: string; + description_subtext?: Array | null; + policy_key?: string | null; title: string; }; diff --git a/clients/admin-ui/src/types/api/models/ConsentMethod.ts b/clients/admin-ui/src/types/api/models/ConsentMethod.ts index b95a109e85..ed8c78cbba 100644 --- a/clients/admin-ui/src/types/api/models/ConsentMethod.ts +++ b/clients/admin-ui/src/types/api/models/ConsentMethod.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum ConsentMethod { BUTTON = "button", REJECT = "reject", diff --git a/clients/admin-ui/src/types/api/models/ConsentPreferences.ts b/clients/admin-ui/src/types/api/models/ConsentPreferences.ts index cb59b0cea7..2a375ea2fe 100644 --- a/clients/admin-ui/src/types/api/models/ConsentPreferences.ts +++ b/clients/admin-ui/src/types/api/models/ConsentPreferences.ts @@ -8,5 +8,5 @@ import type { Consent } from "./Consent"; * Schema for consent preferences. */ export type ConsentPreferences = { - consent?: Array; + consent?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/ConsentPreferencesWithVerificationCode.ts b/clients/admin-ui/src/types/api/models/ConsentPreferencesWithVerificationCode.ts index ff1d33d884..29e01d2f8b 100644 --- a/clients/admin-ui/src/types/api/models/ConsentPreferencesWithVerificationCode.ts +++ b/clients/admin-ui/src/types/api/models/ConsentPreferencesWithVerificationCode.ts @@ -10,9 +10,9 @@ import type { Identity } from "./Identity"; * Schema for consent preferences including the verification code. */ export type ConsentPreferencesWithVerificationCode = { - code?: string; + code?: string | null; consent: Array; - policy_key?: string; - executable_options?: Array; - browser_identity?: Identity; + policy_key?: string | null; + executable_options?: Array | null; + browser_identity?: Identity | null; }; diff --git a/clients/admin-ui/src/types/api/models/ConsentReport.ts b/clients/admin-ui/src/types/api/models/ConsentReport.ts index abc0e9f14c..d0fb06a0d9 100644 --- a/clients/admin-ui/src/types/api/models/ConsentReport.ts +++ b/clients/admin-ui/src/types/api/models/ConsentReport.ts @@ -9,7 +9,7 @@ import type { Identity } from "./Identity"; */ export type ConsentReport = { data_use: string; - data_use_description?: string; + data_use_description?: string | null; opt_in: boolean; has_gpc_flag?: boolean; conflicts_with_gpc?: boolean; diff --git a/clients/admin-ui/src/types/api/models/ConsentReportingSchema.ts b/clients/admin-ui/src/types/api/models/ConsentReportingSchema.ts index 5472ec5b2c..7940b82714 100644 --- a/clients/admin-ui/src/types/api/models/ConsentReportingSchema.ts +++ b/clients/admin-ui/src/types/api/models/ConsentReportingSchema.ts @@ -14,28 +14,28 @@ import type { UserConsentPreference } from "./UserConsentPreference"; */ export type ConsentReportingSchema = { id: string; - privacy_request_id?: string; - email?: string; - phone_number?: string; - external_id?: string; - fides_user_device_id?: string; - secondary_user_ids?: any; + privacy_request_id?: string | null; + email?: string | null; + phone_number?: string | null; + external_id?: string | null; + fides_user_device_id?: string | null; + secondary_user_ids?: null; request_timestamp: string; - request_origin?: RequestOrigin; - request_status?: PrivacyRequestStatus; + request_origin?: RequestOrigin | null; + request_status?: PrivacyRequestStatus | null; request_type: ActionType; - approver_id?: string; - privacy_notice_history_id?: string; - preference?: UserConsentPreference; - user_geography?: string; + approver_id?: string | null; + privacy_notice_history_id?: string | null; + preference?: UserConsentPreference | null; + user_geography?: string | null; affected_system_status: Record; - url_recorded?: string; - user_agent?: string; - experience_config_history_id?: string; - truncated_ip_address?: string; - method?: ConsentMethod; - served_notice_history_id?: string; - notice_name?: string; - tcf_preferences?: any; - property_id?: string; + url_recorded?: string | null; + user_agent?: string | null; + experience_config_history_id?: string | null; + truncated_ip_address?: string | null; + method?: ConsentMethod | null; + served_notice_history_id?: string | null; + notice_name?: string | null; + tcf_preferences?: null; + property_id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/ConsentRequestCreateExtended.ts b/clients/admin-ui/src/types/api/models/ConsentRequestCreateExtended.ts index c47470896c..e41f217e8a 100644 --- a/clients/admin-ui/src/types/api/models/ConsentRequestCreateExtended.ts +++ b/clients/admin-ui/src/types/api/models/ConsentRequestCreateExtended.ts @@ -13,6 +13,6 @@ export type ConsentRequestCreateExtended = { custom_privacy_request_fields?: Record< string, fides__api__schemas__redis_cache__CustomPrivacyRequestField - >; - property_id?: string; + > | null; + property_id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/ConsentableItem.ts b/clients/admin-ui/src/types/api/models/ConsentableItem.ts index 2d424e33fb..5eca1e7706 100644 --- a/clients/admin-ui/src/types/api/models/ConsentableItem.ts +++ b/clients/admin-ui/src/types/api/models/ConsentableItem.ts @@ -9,7 +9,7 @@ export type ConsentableItem = { external_id: string; type: string; name: string; - notice_id?: string; + notice_id?: string | null; children?: Array; - unmapped?: boolean; + unmapped?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/Cookies.ts b/clients/admin-ui/src/types/api/models/Cookies.ts index 923ab591c3..deddb527e4 100644 --- a/clients/admin-ui/src/types/api/models/Cookies.ts +++ b/clients/admin-ui/src/types/api/models/Cookies.ts @@ -7,6 +7,6 @@ */ export type Cookies = { name: string; - path?: string; - domain?: string; + path?: string | null; + domain?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts b/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts index bb002fda88..70373dabe5 100644 --- a/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts +++ b/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts @@ -28,12 +28,12 @@ import type { TimescaleDocsSchema } from "./TimescaleDocsSchema"; * Schema for creating a connection configuration including secrets. */ export type CreateConnectionConfigurationWithSecrets = { - name?: string; - key?: string; + name?: string | null; + key?: string | null; connection_type: ConnectionType; access: AccessLevel; - disabled?: boolean; - description?: string; + disabled?: boolean | null; + description?: string | null; secrets?: | MongoDBDocsSchema | PostgreSQLDocsSchema @@ -53,6 +53,7 @@ export type CreateConnectionConfigurationWithSecrets = { | SovrnDocsSchema | DynamoDBDocsSchema | S3DocsSchema - | ScyllaDocsSchema; - saas_connector_type?: string; + | ScyllaDocsSchema + | null; + saas_connector_type?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecretsExtended.ts b/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecretsExtended.ts index 37c3f02b5a..7a69ccb686 100644 --- a/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecretsExtended.ts +++ b/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecretsExtended.ts @@ -29,12 +29,12 @@ import type { TimescaleDocsSchema } from "./TimescaleDocsSchema"; * An extension of the base fides model with the addition of plus-only fields */ export type CreateConnectionConfigurationWithSecretsExtended = { - name?: string; - key?: string; + name?: string | null; + key?: string | null; connection_type: ConnectionType; access: AccessLevel; - disabled?: boolean; - description?: string; + disabled?: boolean | null; + description?: string | null; secrets?: | MongoDBDocsSchema | PostgreSQLDocsSchema @@ -54,7 +54,8 @@ export type CreateConnectionConfigurationWithSecretsExtended = { | SovrnDocsSchema | DynamoDBDocsSchema | S3DocsSchema - | ScyllaDocsSchema; - saas_connector_type?: string; + | ScyllaDocsSchema + | null; + saas_connector_type?: string | null; enabled_actions: Array; }; diff --git a/clients/admin-ui/src/types/api/models/CurrentStep.ts b/clients/admin-ui/src/types/api/models/CurrentStep.ts index 27fb428eec..cda5c5ad83 100644 --- a/clients/admin-ui/src/types/api/models/CurrentStep.ts +++ b/clients/admin-ui/src/types/api/models/CurrentStep.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum CurrentStep { PRE_WEBHOOKS = "pre_webhooks", ACCESS = "access", diff --git a/clients/admin-ui/src/types/api/models/CustomAssetType.ts b/clients/admin-ui/src/types/api/models/CustomAssetType.ts index 976e2d3d2f..e87b695dbf 100644 --- a/clients/admin-ui/src/types/api/models/CustomAssetType.ts +++ b/clients/admin-ui/src/types/api/models/CustomAssetType.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum CustomAssetType { CUSTOM_FIDES_CSS = "custom-fides.css", } diff --git a/clients/admin-ui/src/types/api/models/CustomFieldDefinition.ts b/clients/admin-ui/src/types/api/models/CustomFieldDefinition.ts index 9d5d9ab189..61b2cfaa46 100644 --- a/clients/admin-ui/src/types/api/models/CustomFieldDefinition.ts +++ b/clients/admin-ui/src/types/api/models/CustomFieldDefinition.ts @@ -7,10 +7,10 @@ import type { ResourceTypes } from "./ResourceTypes"; export type CustomFieldDefinition = { name: string; - description?: string; + description?: string | null; field_type: AllowedTypes; - allow_list_id?: string; + allow_list_id?: string | null; resource_type: ResourceTypes; - field_definition?: string; + field_definition?: string | null; active?: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/CustomFieldDefinitionResponse.ts b/clients/admin-ui/src/types/api/models/CustomFieldDefinitionResponse.ts index 202869ccd2..7ce134e4c8 100644 --- a/clients/admin-ui/src/types/api/models/CustomFieldDefinitionResponse.ts +++ b/clients/admin-ui/src/types/api/models/CustomFieldDefinitionResponse.ts @@ -7,11 +7,11 @@ import type { ResourceTypes } from "./ResourceTypes"; export type CustomFieldDefinitionResponse = { name: string; - description?: string; + description?: string | null; field_type: AllowedTypes; - allow_list_id?: string; + allow_list_id?: string | null; resource_type: ResourceTypes; - field_definition?: string; + field_definition?: string | null; active?: boolean; id: string; created_at: string; diff --git a/clients/admin-ui/src/types/api/models/CustomFieldDefinitionWithId.ts b/clients/admin-ui/src/types/api/models/CustomFieldDefinitionWithId.ts index e8d4591c8a..2a05b45e09 100644 --- a/clients/admin-ui/src/types/api/models/CustomFieldDefinitionWithId.ts +++ b/clients/admin-ui/src/types/api/models/CustomFieldDefinitionWithId.ts @@ -7,11 +7,11 @@ import type { ResourceTypes } from "./ResourceTypes"; export type CustomFieldDefinitionWithId = { name: string; - description?: string; + description?: string | null; field_type: AllowedTypes; - allow_list_id?: string; + allow_list_id?: string | null; resource_type: ResourceTypes; - field_definition?: string; + field_definition?: string | null; active?: boolean; - id?: string; + id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/CustomFieldWithId.ts b/clients/admin-ui/src/types/api/models/CustomFieldWithId.ts index c0a80c7797..781f990e39 100644 --- a/clients/admin-ui/src/types/api/models/CustomFieldWithId.ts +++ b/clients/admin-ui/src/types/api/models/CustomFieldWithId.ts @@ -6,5 +6,5 @@ export type CustomFieldWithId = { resource_id: string; custom_field_definition_id: string; value: string | Array; - id?: string; + id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/DataCategory.ts b/clients/admin-ui/src/types/api/models/DataCategory.ts index 103a1a89c9..7fd7fed349 100644 --- a/clients/admin-ui/src/types/api/models/DataCategory.ts +++ b/clients/admin-ui/src/types/api/models/DataCategory.ts @@ -2,22 +2,19 @@ /* tslint:disable */ /* eslint-disable */ -/** - * The DataCategory resource model. - */ export type DataCategory = { /** * The version of Fideslang in which this label was added. */ - version_added?: string; + version_added?: string | null; /** * The version of Fideslang in which this label was deprecated. */ - version_deprecated?: string; + version_deprecated?: string | null; /** * The new name, if applicable, for this label after deprecation. */ - replaced_by?: string; + replaced_by?: string | null; /** * Denotes whether the resource is part of the default taxonomy or not. */ @@ -30,16 +27,16 @@ export type DataCategory = { * Defines the Organization that this resource belongs to. */ organization_fides_key?: string; - tags?: Array; + tags?: Array | null; /** * Human-Readable name for this resource. */ - name?: string; + name?: string | null; /** * A detailed description of what this resource is. */ - description?: string; - parent_key?: string; + description?: string | null; + parent_key?: string | null; /** * Indicates whether the resource is currently 'active'. */ diff --git a/clients/admin-ui/src/types/api/models/DataFlow.ts b/clients/admin-ui/src/types/api/models/DataFlow.ts index 2352b76cdb..c6db9095ab 100644 --- a/clients/admin-ui/src/types/api/models/DataFlow.ts +++ b/clients/admin-ui/src/types/api/models/DataFlow.ts @@ -19,5 +19,5 @@ export type DataFlow = { /** * An array of data categories describing the data in transit. */ - data_categories?: Array; + data_categories?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/DataSubject.ts b/clients/admin-ui/src/types/api/models/DataSubject.ts index 5365cc411d..76a549f49c 100644 --- a/clients/admin-ui/src/types/api/models/DataSubject.ts +++ b/clients/admin-ui/src/types/api/models/DataSubject.ts @@ -4,22 +4,19 @@ import type { DataSubjectRights } from "./DataSubjectRights"; -/** - * The DataSubject resource model. - */ export type DataSubject = { /** * The version of Fideslang in which this label was added. */ - version_added?: string; + version_added?: string | null; /** * The version of Fideslang in which this label was deprecated. */ - version_deprecated?: string; + version_deprecated?: string | null; /** * The new name, if applicable, for this label after deprecation. */ - replaced_by?: string; + replaced_by?: string | null; /** * Denotes whether the resource is part of the default taxonomy or not. */ @@ -32,15 +29,15 @@ export type DataSubject = { * Defines the Organization that this resource belongs to. */ organization_fides_key?: string; - tags?: Array; + tags?: Array | null; /** * Human-Readable name for this resource. */ - name?: string; + name?: string | null; /** * A detailed description of what this resource is. */ - description?: string; + description?: string | null; /** * * The DataSubjectRights resource model. @@ -50,11 +47,11 @@ export type DataSubject = { * via the set strategy. * */ - rights?: DataSubjectRights; + rights?: DataSubjectRights | null; /** * A boolean value to annotate whether or not automated decisions/profiling exists for the data subject. */ - automated_decisions_or_profiling?: boolean; + automated_decisions_or_profiling?: boolean | null; /** * Indicates whether the resource is currently 'active'. */ diff --git a/clients/admin-ui/src/types/api/models/DataSubjectRights.ts b/clients/admin-ui/src/types/api/models/DataSubjectRights.ts index 693db13a94..103b8d5e1e 100644 --- a/clients/admin-ui/src/types/api/models/DataSubjectRights.ts +++ b/clients/admin-ui/src/types/api/models/DataSubjectRights.ts @@ -20,5 +20,5 @@ export type DataSubjectRights = { /** * A list of valid data subject rights to be used when applying data rights to a data subject via a strategy. */ - values?: Array; + values?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/DataUse.ts b/clients/admin-ui/src/types/api/models/DataUse.ts index d458682d32..4df688916f 100644 --- a/clients/admin-ui/src/types/api/models/DataUse.ts +++ b/clients/admin-ui/src/types/api/models/DataUse.ts @@ -2,22 +2,19 @@ /* tslint:disable */ /* eslint-disable */ -/** - * The DataUse resource model. - */ export type DataUse = { /** * The version of Fideslang in which this label was added. */ - version_added?: string; + version_added?: string | null; /** * The version of Fideslang in which this label was deprecated. */ - version_deprecated?: string; + version_deprecated?: string | null; /** * The new name, if applicable, for this label after deprecation. */ - replaced_by?: string; + replaced_by?: string | null; /** * Denotes whether the resource is part of the default taxonomy or not. */ @@ -30,16 +27,16 @@ export type DataUse = { * Defines the Organization that this resource belongs to. */ organization_fides_key?: string; - tags?: Array; + tags?: Array | null; /** * Human-Readable name for this resource. */ - name?: string; + name?: string | null; /** * A detailed description of what this resource is. */ - description?: string; - parent_key?: string; + description?: string | null; + parent_key?: string | null; /** * Indicates whether the resource is currently 'active'. */ diff --git a/clients/admin-ui/src/types/api/models/Database.ts b/clients/admin-ui/src/types/api/models/Database.ts index e6a08184eb..d468986ad3 100644 --- a/clients/admin-ui/src/types/api/models/Database.ts +++ b/clients/admin-ui/src/types/api/models/Database.ts @@ -5,22 +5,19 @@ import type { Classification } from "./Classification"; import type { DiffStatus } from "./DiffStatus"; -/** - * Base API model that represents a staged resource, fields common to all types of staged resources - */ export type Database = { urn: string; user_assigned_data_categories?: Array; - name?: string; - description?: string; - monitor_config_id?: string; - updated_at?: string; - source_modified?: string; + name?: string | null; + description?: string | null; + monitor_config_id?: string | null; + updated_at?: string | null; + source_modified?: string | null; classifications?: Array; /** * The diff status of the staged resource */ - diff_status?: DiffStatus; + diff_status?: DiffStatus | null; /** * Represents the aggregate counts of diff statuses of the staged resource's children. This is computed 'on-demand', i.e. a specific instance method must be invoked to populate the field. */ diff --git a/clients/admin-ui/src/types/api/models/DatabaseHealthCheck.ts b/clients/admin-ui/src/types/api/models/DatabaseHealthCheck.ts index 3c23429707..6fc7330a81 100644 --- a/clients/admin-ui/src/types/api/models/DatabaseHealthCheck.ts +++ b/clients/admin-ui/src/types/api/models/DatabaseHealthCheck.ts @@ -7,5 +7,5 @@ */ export type DatabaseHealthCheck = { database: string; - database_revision?: string; + database_revision?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/DatamapReport.ts b/clients/admin-ui/src/types/api/models/DatamapReport.ts index 06db2982e7..5283343ba5 100644 --- a/clients/admin-ui/src/types/api/models/DatamapReport.ts +++ b/clients/admin-ui/src/types/api/models/DatamapReport.ts @@ -3,50 +3,49 @@ /* eslint-disable */ export type DatamapReport = { - administrating_department?: string; - cookie_max_age_seconds?: number; + administrating_department?: string | null; + cookie_max_age_seconds?: number | null; cookie_refresh: boolean; - cookies?: Array; - data_categories?: string | Array; - system_undeclared_data_categories?: Array; - data_use_undeclared_data_categories?: Array; - data_security_practices?: string; - data_shared_with_third_parties?: boolean; - data_stewards?: Array; - data_subjects?: Array; - data_uses?: string | Array; - declaration_name?: string; - description?: string; + data_categories?: string | Array | null; + system_undeclared_data_categories?: Array | null; + data_use_undeclared_data_categories?: Array | null; + data_security_practices?: string | null; + data_shared_with_third_parties?: boolean | null; + data_stewards?: Array | null; + data_subjects?: Array | null; + data_uses?: string | Array | null; + declaration_name?: string | null; + description?: string | null; does_international_transfers: boolean; - dpa_location?: string; - dpo?: string; - egress?: Array; + dpa_location?: string | null; + dpo?: string | null; + egress?: Array | null; exempt_from_privacy_regulations: boolean; - features?: Array; + features?: Array | null; fides_key: string; - flexible_legal_basis_for_processing?: boolean; - impact_assessment_location?: string; - ingress?: Array; - joint_controller_info?: string; - legal_address?: string; - legal_basis_for_processing?: string; - legal_basis_for_profiling?: Array; - legal_basis_for_transfers?: Array; - legal_name?: string; - legitimate_interest_disclosure_url?: string; - link_to_processor_contract?: string; - privacy_policy?: string; + flexible_legal_basis_for_processing?: boolean | null; + impact_assessment_location?: string | null; + ingress?: Array | null; + joint_controller_info?: string | null; + legal_address?: string | null; + legal_basis_for_processing?: string | null; + legal_basis_for_profiling?: Array | null; + legal_basis_for_transfers?: Array | null; + legal_name?: string | null; + legitimate_interest_disclosure_url?: string | null; + link_to_processor_contract?: string | null; + privacy_policy?: string | null; processes_personal_data: boolean; - reason_for_exemption?: string; + reason_for_exemption?: string | null; requires_data_protection_assessments: boolean; - responsibility?: Array; - retention_period?: string; - shared_categories?: Array; - special_category_legal_basis?: string; - system_dependencies?: string; + responsibility?: Array | null; + retention_period?: string | null; + shared_categories?: Array | null; + special_category_legal_basis?: string | null; + system_dependencies?: string | null; system_name: string; - third_country_safeguards?: string; - third_parties?: string; + third_country_safeguards?: string | null; + third_parties?: string | null; uses_cookies: boolean; uses_non_cookie_access: boolean; uses_profiling: boolean; diff --git a/clients/admin-ui/src/types/api/models/Dataset.ts b/clients/admin-ui/src/types/api/models/Dataset.ts index 3500d3cc7e..d262cc3523 100644 --- a/clients/admin-ui/src/types/api/models/Dataset.ts +++ b/clients/admin-ui/src/types/api/models/Dataset.ts @@ -17,23 +17,23 @@ export type Dataset = { * Defines the Organization that this resource belongs to. */ organization_fides_key?: string; - tags?: Array; + tags?: Array | null; /** * Human-Readable name for this resource. */ - name?: string; + name?: string | null; /** * A detailed description of what this resource is. */ - description?: string; + description?: string | null; /** * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. */ - meta?: any; + meta?: null; /** * Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset. */ - data_categories?: Array; + data_categories?: Array | null; /** * * The DatasetMetadata resource model. @@ -41,7 +41,7 @@ export type Dataset = { * Object used to hold application specific metadata for a dataset * */ - fides_meta?: DatasetMetadata; + fides_meta?: DatasetMetadata | null; /** * An array of objects that describe the Dataset's collections. */ diff --git a/clients/admin-ui/src/types/api/models/DatasetCollection.ts b/clients/admin-ui/src/types/api/models/DatasetCollection.ts index 44b96847ea..f62a8e4f06 100644 --- a/clients/admin-ui/src/types/api/models/DatasetCollection.ts +++ b/clients/admin-ui/src/types/api/models/DatasetCollection.ts @@ -18,14 +18,14 @@ export type DatasetCollection = { /** * A detailed description of what this resource is. */ - description?: string; + description?: string | null; /** * Array of Data Category resources identified by `fides_key`, that apply to all fields in the collection. */ - data_categories?: Array; + data_categories?: Array | null; /** * An array of objects that describe the collection's fields. */ fields: Array; - fides_meta?: CollectionMeta; + fides_meta?: CollectionMeta | null; }; diff --git a/clients/admin-ui/src/types/api/models/DatasetConfigCtlDataset.ts b/clients/admin-ui/src/types/api/models/DatasetConfigCtlDataset.ts index 444f2a212f..9da28b241a 100644 --- a/clients/admin-ui/src/types/api/models/DatasetConfigCtlDataset.ts +++ b/clients/admin-ui/src/types/api/models/DatasetConfigCtlDataset.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type DatasetConfigCtlDataset = { fides_key: string; ctl_dataset_fides_key: string; diff --git a/clients/admin-ui/src/types/api/models/DatasetField.ts b/clients/admin-ui/src/types/api/models/DatasetField.ts index 593346a505..9c4a4c4691 100644 --- a/clients/admin-ui/src/types/api/models/DatasetField.ts +++ b/clients/admin-ui/src/types/api/models/DatasetField.ts @@ -17,14 +17,14 @@ export type DatasetField = { /** * A detailed description of what this resource is. */ - description?: string; + description?: string | null; /** * Arrays of Data Categories, identified by `fides_key`, that applies to this field. */ - data_categories?: Array; - fides_meta?: FidesMeta; + data_categories?: Array | null; + fides_meta?: FidesMeta | null; /** * An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases). */ - fields?: Array; + fields?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/DatasetMetadata.ts b/clients/admin-ui/src/types/api/models/DatasetMetadata.ts index bd68adc9db..0fcf9be9cb 100644 --- a/clients/admin-ui/src/types/api/models/DatasetMetadata.ts +++ b/clients/admin-ui/src/types/api/models/DatasetMetadata.ts @@ -8,6 +8,6 @@ * Object used to hold application specific metadata for a dataset */ export type DatasetMetadata = { - resource_id?: string; - after?: Array; + resource_id?: string | null; + after?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/DatasetSchema.ts b/clients/admin-ui/src/types/api/models/DatasetSchema.ts index 2aec0a53c5..4b436cc17f 100644 --- a/clients/admin-ui/src/types/api/models/DatasetSchema.ts +++ b/clients/admin-ui/src/types/api/models/DatasetSchema.ts @@ -4,5 +4,5 @@ export type DatasetSchema = { fides_key: string; - name?: string; + name?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/DatasetTraversalDetails.ts b/clients/admin-ui/src/types/api/models/DatasetTraversalDetails.ts index 6541c40eff..d2993f9016 100644 --- a/clients/admin-ui/src/types/api/models/DatasetTraversalDetails.ts +++ b/clients/admin-ui/src/types/api/models/DatasetTraversalDetails.ts @@ -8,5 +8,5 @@ */ export type DatasetTraversalDetails = { is_traversable: boolean; - msg?: string; + msg: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/DenyPrivacyRequests.ts b/clients/admin-ui/src/types/api/models/DenyPrivacyRequests.ts index 178f50cbd5..1f883b3463 100644 --- a/clients/admin-ui/src/types/api/models/DenyPrivacyRequests.ts +++ b/clients/admin-ui/src/types/api/models/DenyPrivacyRequests.ts @@ -7,5 +7,5 @@ */ export type DenyPrivacyRequests = { request_ids: Array; - reason?: string; + reason?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/DictionaryStatus.ts b/clients/admin-ui/src/types/api/models/DictionaryStatus.ts index e7508719f1..af9ac8f67d 100644 --- a/clients/admin-ui/src/types/api/models/DictionaryStatus.ts +++ b/clients/admin-ui/src/types/api/models/DictionaryStatus.ts @@ -9,6 +9,6 @@ import type { ServiceHealth } from "./ServiceHealth"; */ export type DictionaryStatus = { enabled?: boolean; - service_health?: ServiceHealth; - service_error?: string; + service_health?: ServiceHealth | null; + service_error?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/DiffStatus.ts b/clients/admin-ui/src/types/api/models/DiffStatus.ts index 9102552a60..a7ea68688f 100644 --- a/clients/admin-ui/src/types/api/models/DiffStatus.ts +++ b/clients/admin-ui/src/types/api/models/DiffStatus.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum DiffStatus { ADDITION = "addition", REMOVAL = "removal", diff --git a/clients/admin-ui/src/types/api/models/DrpDataRightsResponse.ts b/clients/admin-ui/src/types/api/models/DrpDataRightsResponse.ts index 18f808712a..ec31318a37 100644 --- a/clients/admin-ui/src/types/api/models/DrpDataRightsResponse.ts +++ b/clients/admin-ui/src/types/api/models/DrpDataRightsResponse.ts @@ -9,7 +9,7 @@ import type { DrpAction } from "./DrpAction"; */ export type DrpDataRightsResponse = { version: string; - api_base?: string; + api_base?: string | null; actions: Array; - user_relationships?: Array; + user_relationships?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/DrpPrivacyRequestCreate.ts b/clients/admin-ui/src/types/api/models/DrpPrivacyRequestCreate.ts index 7bf91178e3..e98d990fb7 100644 --- a/clients/admin-ui/src/types/api/models/DrpPrivacyRequestCreate.ts +++ b/clients/admin-ui/src/types/api/models/DrpPrivacyRequestCreate.ts @@ -11,9 +11,9 @@ import type { DrpRegime } from "./DrpRegime"; */ export type DrpPrivacyRequestCreate = { meta: DrpMeta; - regime?: DrpRegime; + regime?: DrpRegime | null; exercise: Array; - relationships?: Array; + relationships?: Array | null; identity: string; - status_callback?: string; + status_callback?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/DrpRevokeRequest.ts b/clients/admin-ui/src/types/api/models/DrpRevokeRequest.ts index 8ecd0a20dd..74ea59b345 100644 --- a/clients/admin-ui/src/types/api/models/DrpRevokeRequest.ts +++ b/clients/admin-ui/src/types/api/models/DrpRevokeRequest.ts @@ -7,5 +7,5 @@ */ export type DrpRevokeRequest = { request_id: string; - reason?: string; + reason?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/DryRunDatasetResponse.ts b/clients/admin-ui/src/types/api/models/DryRunDatasetResponse.ts index feb3ebab95..4d55cc5f44 100644 --- a/clients/admin-ui/src/types/api/models/DryRunDatasetResponse.ts +++ b/clients/admin-ui/src/types/api/models/DryRunDatasetResponse.ts @@ -9,5 +9,5 @@ import type { CollectionAddressResponse } from "./CollectionAddressResponse"; */ export type DryRunDatasetResponse = { collectionAddress: CollectionAddressResponse; - query?: any; + query: any; }; diff --git a/clients/admin-ui/src/types/api/models/DynamoDBMonitorParams.ts b/clients/admin-ui/src/types/api/models/DynamoDBMonitorParams.ts index 7d9736d865..76c31d0f7d 100644 --- a/clients/admin-ui/src/types/api/models/DynamoDBMonitorParams.ts +++ b/clients/admin-ui/src/types/api/models/DynamoDBMonitorParams.ts @@ -6,5 +6,5 @@ export type DynamoDBMonitorParams = { /** * Whether the DynamoDB monitor should put all tables found in Dynamo into a single Fides Dataset, or one Dataset per Dynamo table */ - single_dataset?: boolean; + single_dataset?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/EditableMonitorConfig.ts b/clients/admin-ui/src/types/api/models/EditableMonitorConfig.ts index c444290d65..1b71b47e00 100644 --- a/clients/admin-ui/src/types/api/models/EditableMonitorConfig.ts +++ b/clients/admin-ui/src/types/api/models/EditableMonitorConfig.ts @@ -12,25 +12,25 @@ import type { S3MonitorParams } from "./S3MonitorParams"; */ export type EditableMonitorConfig = { name: string; - key?: string; + key?: string | null; connection_config_key: string; classify_params: MonitorClassifyParams; /** * The datasource specific parameters, specified in a dictionary */ - datasource_params?: DynamoDBMonitorParams | S3MonitorParams; + datasource_params?: DynamoDBMonitorParams | S3MonitorParams | null; /** * The databases that the monitor is scoped to actively monitor */ databases?: Array; + execution_start_date?: string | null; + execution_frequency?: MonitorFrequency | null; /** * The databases that the monitor should exclude from monitoring */ excluded_databases?: Array; - execution_start_date?: string; - execution_frequency?: MonitorFrequency; /** * Indicates whether the monitor is enabled or not. Disabled monitors won't be executed */ - enabled?: boolean; + enabled?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts b/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts index dbd507a4ec..afd1154db5 100644 --- a/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts @@ -10,6 +10,6 @@ import type { AdvancedSettings } from "./AdvancedSettings"; export type EmailDocsSchema = { third_party_vendor_name: string; recipient_email_address: string; - test_email_address?: string; + test_email_address?: string | null; advanced_settings?: AdvancedSettings; }; diff --git a/clients/admin-ui/src/types/api/models/EmbeddedPurpose.ts b/clients/admin-ui/src/types/api/models/EmbeddedPurpose.ts index faf672f2f3..8c2a511c98 100644 --- a/clients/admin-ui/src/types/api/models/EmbeddedPurpose.ts +++ b/clients/admin-ui/src/types/api/models/EmbeddedPurpose.ts @@ -8,5 +8,5 @@ export type EmbeddedPurpose = { id: number; name: string; - retention_period?: string; + retention_period?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/Evaluation.ts b/clients/admin-ui/src/types/api/models/Evaluation.ts index 9f4e6ac176..6915acf1b3 100644 --- a/clients/admin-ui/src/types/api/models/Evaluation.ts +++ b/clients/admin-ui/src/types/api/models/Evaluation.ts @@ -15,9 +15,6 @@ export type Evaluation = { * A uuid generated for each unique evaluation. */ fides_key: string; - /** - * The model for possible evaluation results. - */ status: StatusEnum; /** * The model for violations within an evaluation. diff --git a/clients/admin-ui/src/types/api/models/ExecutionAndAuditLogResponse.ts b/clients/admin-ui/src/types/api/models/ExecutionAndAuditLogResponse.ts index f41b8bce20..6c0b3d4623 100644 --- a/clients/admin-ui/src/types/api/models/ExecutionAndAuditLogResponse.ts +++ b/clients/admin-ui/src/types/api/models/ExecutionAndAuditLogResponse.ts @@ -12,12 +12,12 @@ import type { FieldsAffectedResponse } from "./FieldsAffectedResponse"; * associated with a PrivacyRequest */ export type ExecutionAndAuditLogResponse = { - connection_key?: string; - collection_name?: string; - fields_affected?: Array; - message?: string; - action_type?: ActionType; - status?: ExecutionLogStatus | AuditLogAction; - updated_at?: string; - user_id?: string; + connection_key?: string | null; + collection_name?: string | null; + fields_affected?: Array | null; + message?: string | null; + action_type?: ActionType | null; + status?: ExecutionLogStatus | AuditLogAction | string | null; + updated_at?: string | null; + user_id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/ExecutionApplicationConfig.ts b/clients/admin-ui/src/types/api/models/ExecutionApplicationConfig.ts index f88c9ba599..509d665d14 100644 --- a/clients/admin-ui/src/types/api/models/ExecutionApplicationConfig.ts +++ b/clients/admin-ui/src/types/api/models/ExecutionApplicationConfig.ts @@ -2,11 +2,8 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type ExecutionApplicationConfig = { - subject_identity_verification_required?: boolean; - disable_consent_identity_verification?: boolean; - require_manual_request_approval?: boolean; + subject_identity_verification_required?: boolean | null; + disable_consent_identity_verification?: boolean | null; + require_manual_request_approval?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/ExecutionLogDetailResponse.ts b/clients/admin-ui/src/types/api/models/ExecutionLogDetailResponse.ts index a7009e1498..77f255f782 100644 --- a/clients/admin-ui/src/types/api/models/ExecutionLogDetailResponse.ts +++ b/clients/admin-ui/src/types/api/models/ExecutionLogDetailResponse.ts @@ -10,12 +10,12 @@ import type { FieldsAffectedResponse } from "./FieldsAffectedResponse"; * Schema for the detailed ExecutionLogs when accessed directly */ export type ExecutionLogDetailResponse = { - collection_name?: string; - fields_affected?: Array; - message?: string; - action_type: ActionType; status: ExecutionLogStatus; - updated_at?: string; - connection_key?: string; - dataset_name?: string; + collection_name?: string | null; + fields_affected?: Array | null; + message?: string | null; + action_type: ActionType; + updated_at?: string | null; + connection_key?: string | null; + dataset_name?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/ExperienceConfigCreate.ts b/clients/admin-ui/src/types/api/models/ExperienceConfigCreate.ts index 080b5c76f3..cde3b792ca 100644 --- a/clients/admin-ui/src/types/api/models/ExperienceConfigCreate.ts +++ b/clients/admin-ui/src/types/api/models/ExperienceConfigCreate.ts @@ -15,12 +15,12 @@ import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; */ export type ExperienceConfigCreate = { name: string; - disabled?: boolean; - dismissable?: boolean; - show_layer1_notices?: boolean; - layer1_button_options?: Layer1ButtonOption; - allow_language_selection?: boolean; - auto_detect_language?: boolean; + disabled?: boolean | null; + dismissable?: boolean | null; + show_layer1_notices?: boolean | null; + layer1_button_options?: Layer1ButtonOption | null; + allow_language_selection?: boolean | null; + auto_detect_language?: boolean | null; regions?: Array; component: ComponentType; privacy_notice_ids?: Array; diff --git a/clients/admin-ui/src/types/api/models/ExperienceConfigResponse.ts b/clients/admin-ui/src/types/api/models/ExperienceConfigResponse.ts index 90a346b016..ff09cafd77 100644 --- a/clients/admin-ui/src/types/api/models/ExperienceConfigResponse.ts +++ b/clients/admin-ui/src/types/api/models/ExperienceConfigResponse.ts @@ -14,17 +14,17 @@ import type { PrivacyNoticeResponse } from "./PrivacyNoticeResponse"; */ export type ExperienceConfigResponse = { name: string; - disabled?: boolean; - dismissable?: boolean; - show_layer1_notices?: boolean; - layer1_button_options?: Layer1ButtonOption; - allow_language_selection?: boolean; - auto_detect_language?: boolean; + disabled?: boolean | null; + dismissable?: boolean | null; + show_layer1_notices?: boolean | null; + layer1_button_options?: Layer1ButtonOption | null; + allow_language_selection?: boolean | null; + auto_detect_language?: boolean | null; regions: Array; id: string; created_at: string; updated_at: string; - origin?: string; + origin?: string | null; component: ComponentType; privacy_notices?: Array; translations?: Array; diff --git a/clients/admin-ui/src/types/api/models/ExperienceConfigResponseNoNotices.ts b/clients/admin-ui/src/types/api/models/ExperienceConfigResponseNoNotices.ts index 6e2bc81f97..3c25a43481 100644 --- a/clients/admin-ui/src/types/api/models/ExperienceConfigResponseNoNotices.ts +++ b/clients/admin-ui/src/types/api/models/ExperienceConfigResponseNoNotices.ts @@ -21,62 +21,62 @@ export type ExperienceConfigResponseNoNotices = { /** * Accept or confirmation button label */ - accept_button_label?: string; + accept_button_label?: string | null; /** * Acknowledge button label for notice only */ - acknowledge_button_label?: string; + acknowledge_button_label?: string | null; /** * Banner title */ - banner_title?: string; + banner_title?: string | null; /** * Whether the given translation is the default */ - is_default?: boolean; + is_default?: boolean | null; /** * Privacy policy link label */ - privacy_policy_link_label?: string; + privacy_policy_link_label?: string | null; /** * Modal link label */ - modal_link_label?: string; + modal_link_label?: string | null; /** * Privacy policy URL */ - privacy_policy_url?: string; + privacy_policy_url?: string | null; /** * Privacy preferences link label */ - privacy_preferences_link_label?: string; + privacy_preferences_link_label?: string | null; /** * Reject button label */ - reject_button_label?: string; + reject_button_label?: string | null; /** * Save button label */ - save_button_label?: string; + save_button_label?: string | null; /** * Overall title */ - title?: string; + title?: string | null; /** * Banner description. HTML descriptions are supported so links can be included. */ - banner_description?: string; + banner_description?: string | null; /** * Overall description - used for banner as well if applicable. HTML descriptions are supported so links can be included. */ - description?: string; + description?: string | null; name: string; - disabled?: boolean; - dismissable?: boolean; - show_layer1_notices?: boolean; - layer1_button_options?: Layer1ButtonOption; - allow_language_selection?: boolean; - auto_detect_language?: boolean; + disabled?: boolean | null; + dismissable?: boolean | null; + show_layer1_notices?: boolean | null; + layer1_button_options?: Layer1ButtonOption | null; + allow_language_selection?: boolean | null; + auto_detect_language?: boolean | null; regions: Array; id: string; created_at: string; diff --git a/clients/admin-ui/src/types/api/models/ExperienceConfigUpdate.ts b/clients/admin-ui/src/types/api/models/ExperienceConfigUpdate.ts index 87eef69aae..6705ed489b 100644 --- a/clients/admin-ui/src/types/api/models/ExperienceConfigUpdate.ts +++ b/clients/admin-ui/src/types/api/models/ExperienceConfigUpdate.ts @@ -18,13 +18,13 @@ import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; * schema after patch dry updates are applied. */ export type ExperienceConfigUpdate = { - name?: string; - disabled?: boolean; - dismissable?: boolean; - show_layer1_notices?: boolean; - layer1_button_options?: Layer1ButtonOption; - allow_language_selection?: boolean; - auto_detect_language?: boolean; + name?: string | null; + disabled?: boolean | null; + dismissable?: boolean | null; + show_layer1_notices?: boolean | null; + layer1_button_options?: Layer1ButtonOption | null; + allow_language_selection?: boolean | null; + auto_detect_language?: boolean | null; regions: Array; translations: Array; privacy_notice_ids: Array; diff --git a/clients/admin-ui/src/types/api/models/ExperienceMeta.ts b/clients/admin-ui/src/types/api/models/ExperienceMeta.ts index 5e4ecc5c55..1fc8b657f4 100644 --- a/clients/admin-ui/src/types/api/models/ExperienceMeta.ts +++ b/clients/admin-ui/src/types/api/models/ExperienceMeta.ts @@ -11,15 +11,15 @@ export type ExperienceMeta = { /** * A hashed value that can be compared to previously-fetched hash values to determine if the Experience has meaningfully changed */ - version_hash?: string; + version_hash?: string | null; /** * The fides string (TC String + AC String) corresponding to a user opting in to all available options */ - accept_all_fides_string?: string; - accept_all_fides_mobile_data?: TCMobileData; + accept_all_fides_string?: string | null; + accept_all_fides_mobile_data?: TCMobileData | null; /** * The fides string (TC String + AC String) corresponding to a user opting out of all available options */ - reject_all_fides_string?: string; - reject_all_fides_mobile_data?: TCMobileData; + reject_all_fides_string?: string | null; + reject_all_fides_mobile_data?: TCMobileData | null; }; diff --git a/clients/admin-ui/src/types/api/models/ExperienceTranslation.ts b/clients/admin-ui/src/types/api/models/ExperienceTranslation.ts index 4aa7d021e0..27097a5891 100644 --- a/clients/admin-ui/src/types/api/models/ExperienceTranslation.ts +++ b/clients/admin-ui/src/types/api/models/ExperienceTranslation.ts @@ -12,53 +12,53 @@ export type ExperienceTranslation = { /** * Accept or confirmation button label */ - accept_button_label?: string; + accept_button_label?: string | null; /** * Acknowledge button label for notice only */ - acknowledge_button_label?: string; + acknowledge_button_label?: string | null; /** * Banner title */ - banner_title?: string; + banner_title?: string | null; /** * Whether the given translation is the default */ - is_default?: boolean; + is_default?: boolean | null; /** * Privacy policy link label */ - privacy_policy_link_label?: string; + privacy_policy_link_label?: string | null; /** * Modal link label */ - modal_link_label?: string; + modal_link_label?: string | null; /** * Privacy policy URL */ - privacy_policy_url?: string; + privacy_policy_url?: string | null; /** * Privacy preferences link label */ - privacy_preferences_link_label?: string; + privacy_preferences_link_label?: string | null; /** * Reject button label */ - reject_button_label?: string; + reject_button_label?: string | null; /** * Save button label */ - save_button_label?: string; + save_button_label?: string | null; /** * Overall title */ - title?: string; + title?: string | null; /** * Banner description. HTML descriptions are supported so links can be included. */ - banner_description?: string; + banner_description?: string | null; /** * Overall description - used for banner as well if applicable. HTML descriptions are supported so links can be included. */ - description?: string; + description?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/ExperienceTranslationCreate.ts b/clients/admin-ui/src/types/api/models/ExperienceTranslationCreate.ts index 7bb9e5e791..38bdd52467 100644 --- a/clients/admin-ui/src/types/api/models/ExperienceTranslationCreate.ts +++ b/clients/admin-ui/src/types/api/models/ExperienceTranslationCreate.ts @@ -12,47 +12,47 @@ export type ExperienceTranslationCreate = { /** * Accept or confirmation button label */ - accept_button_label?: string; + accept_button_label?: string | null; /** * Acknowledge button label for notice only */ - acknowledge_button_label?: string; + acknowledge_button_label?: string | null; /** * Banner title */ - banner_title?: string; + banner_title?: string | null; /** * Whether the given translation is the default */ - is_default?: boolean; + is_default?: boolean | null; /** * Privacy policy link label */ - privacy_policy_link_label?: string; + privacy_policy_link_label?: string | null; /** * Modal link label */ - modal_link_label?: string; + modal_link_label?: string | null; /** * Privacy policy URL */ - privacy_policy_url?: string; + privacy_policy_url?: string | null; /** * Privacy preferences link label */ - privacy_preferences_link_label?: string; + privacy_preferences_link_label?: string | null; /** * Reject button label */ - reject_button_label?: string; + reject_button_label?: string | null; /** * Save button label */ - save_button_label?: string; + save_button_label?: string | null; title: string; /** * Banner description. HTML descriptions are supported so links can be included. */ - banner_description?: string; + banner_description?: string | null; description: string; }; diff --git a/clients/admin-ui/src/types/api/models/ExperienceTranslationResponse.ts b/clients/admin-ui/src/types/api/models/ExperienceTranslationResponse.ts index 8418eaf247..e4cdab5200 100644 --- a/clients/admin-ui/src/types/api/models/ExperienceTranslationResponse.ts +++ b/clients/admin-ui/src/types/api/models/ExperienceTranslationResponse.ts @@ -12,55 +12,55 @@ export type ExperienceTranslationResponse = { /** * Accept or confirmation button label */ - accept_button_label?: string; + accept_button_label?: string | null; /** * Acknowledge button label for notice only */ - acknowledge_button_label?: string; + acknowledge_button_label?: string | null; /** * Banner title */ - banner_title?: string; + banner_title?: string | null; /** * Whether the given translation is the default */ - is_default?: boolean; + is_default?: boolean | null; /** * Privacy policy link label */ - privacy_policy_link_label?: string; + privacy_policy_link_label?: string | null; /** * Modal link label */ - modal_link_label?: string; + modal_link_label?: string | null; /** * Privacy policy URL */ - privacy_policy_url?: string; + privacy_policy_url?: string | null; /** * Privacy preferences link label */ - privacy_preferences_link_label?: string; + privacy_preferences_link_label?: string | null; /** * Reject button label */ - reject_button_label?: string; + reject_button_label?: string | null; /** * Save button label */ - save_button_label?: string; + save_button_label?: string | null; /** * Overall title */ - title?: string; + title?: string | null; /** * Banner description. HTML descriptions are supported so links can be included. */ - banner_description?: string; + banner_description?: string | null; /** * Overall description - used for banner as well if applicable. HTML descriptions are supported so links can be included. */ - description?: string; + description?: string | null; /** * The versioned artifact of the translation and its Experience Config. Should be supplied when saving privacy preferences for additional context. */ diff --git a/clients/admin-ui/src/types/api/models/ExternalDatasetReference.ts b/clients/admin-ui/src/types/api/models/ExternalDatasetReference.ts index 84a2a62124..3eb6e482fd 100644 --- a/clients/admin-ui/src/types/api/models/ExternalDatasetReference.ts +++ b/clients/admin-ui/src/types/api/models/ExternalDatasetReference.ts @@ -4,6 +4,6 @@ export type ExternalDatasetReference = { name: string; - label?: string; - description?: string; + label?: string | null; + description?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts b/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts index 39e2fd943e..9d5b1fa7c1 100644 --- a/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts +++ b/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts @@ -10,5 +10,5 @@ import type { EdgeDirection } from "./EdgeDirection"; export type FidesDatasetReference = { dataset: string; field: string; - direction?: EdgeDirection; + direction?: EdgeDirection | null; }; diff --git a/clients/admin-ui/src/types/api/models/FidesDocsSchema.ts b/clients/admin-ui/src/types/api/models/FidesDocsSchema.ts index ebdfdc728c..6c3770524f 100644 --- a/clients/admin-ui/src/types/api/models/FidesDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/FidesDocsSchema.ts @@ -9,6 +9,6 @@ export type FidesDocsSchema = { uri: string; username: string; password: string; - polling_timeout?: number; - polling_interval?: number; + polling_timeout?: number | null; + polling_interval?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/FidesMeta.ts b/clients/admin-ui/src/types/api/models/FidesMeta.ts index 541466c226..1ee5f359ea 100644 --- a/clients/admin-ui/src/types/api/models/FidesMeta.ts +++ b/clients/admin-ui/src/types/api/models/FidesMeta.ts @@ -11,29 +11,29 @@ export type FidesMeta = { /** * Fields that current field references or is referenced by. Used for drawing the edges of a DSR graph. */ - references?: Array; + references?: Array | null; /** * The type of the identity data that should be used to query this collection for a DSR. */ - identity?: string; + identity?: string | null; /** * Whether the current field can be considered a primary key of the current collection */ - primary_key?: boolean; + primary_key?: boolean | null; /** * Optionally specify the data type. Fides will attempt to cast values to this type when querying. */ - data_type?: string; + data_type?: string | null; /** * Optionally specify the allowable field length. Fides will not generate values that exceed this size. */ - length?: number; + length?: number | null; /** * Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False. */ - return_all_elements?: boolean; + return_all_elements?: boolean | null; /** * Optionally specify if a field is read-only, meaning it can't be updated or deleted. */ - read_only?: boolean; + read_only?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/Field.ts b/clients/admin-ui/src/types/api/models/Field.ts index 75938d15ad..67ea1d975b 100644 --- a/clients/admin-ui/src/types/api/models/Field.ts +++ b/clients/admin-ui/src/types/api/models/Field.ts @@ -6,31 +6,39 @@ import type { Classification } from "./Classification"; import type { DiffStatus } from "./DiffStatus"; /** - * Base API model that represents a staged resource, fields common to all types of staged resources + * The Field model is also used to represent "sub-fields" that are nested under a + * top-level field in a data source. + * + * In these cases, the `name` attribute on the Field should be the full "path" + * to the sub-field, minus the name of the top-level field. + * + * The top-level field name for a given sub-field is stored in its own attribute, + * which is only populated if the Field is a sub-field. */ export type Field = { urn: string; user_assigned_data_categories?: Array; - name?: string; - description?: string; - monitor_config_id?: string; - updated_at?: string; - source_modified?: string; + name?: string | null; + description?: string | null; + monitor_config_id?: string | null; + updated_at?: string | null; + source_modified?: string | null; classifications?: Array; /** * The diff status of the staged resource */ - diff_status?: DiffStatus; + diff_status?: DiffStatus | null; /** * Represents the aggregate counts of diff statuses of the staged resource's children. This is computed 'on-demand', i.e. a specific instance method must be invoked to populate the field. */ child_diff_statuses?: Record; - database_name?: string; + database_name?: string | null; schema_name: string; parent_table_urn: string; table_name: string; - data_type?: string; + data_type?: string | null; sub_field_urns?: Array; direct_child_urns?: Array; - top_level_field_name?: string; + top_level_field_name?: string | null; + source_data_type?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/FieldsAffectedResponse.ts b/clients/admin-ui/src/types/api/models/FieldsAffectedResponse.ts index 347b9d72b4..66d26daccc 100644 --- a/clients/admin-ui/src/types/api/models/FieldsAffectedResponse.ts +++ b/clients/admin-ui/src/types/api/models/FieldsAffectedResponse.ts @@ -6,7 +6,7 @@ * Schema detailing the individual fields affected by a particular query detailed in the ExecutionLog */ export type FieldsAffectedResponse = { - path?: string; - field_name?: string; - data_categories?: Array; + path: string | null; + field_name: string | null; + data_categories: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/GPPApplicationConfig.ts b/clients/admin-ui/src/types/api/models/GPPApplicationConfig.ts index d2bd7502eb..1b72f4dd0e 100644 --- a/clients/admin-ui/src/types/api/models/GPPApplicationConfig.ts +++ b/clients/admin-ui/src/types/api/models/GPPApplicationConfig.ts @@ -4,13 +4,10 @@ import type { GPPUSApproach } from "./GPPUSApproach"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type GPPApplicationConfig = { - us_approach?: GPPUSApproach; - mspa_service_provider_mode?: boolean; - mspa_opt_out_option_mode?: boolean; - mspa_covered_transactions?: boolean; - enable_tcfeu_string?: boolean; + us_approach?: GPPUSApproach | null; + mspa_service_provider_mode?: boolean | null; + mspa_opt_out_option_mode?: boolean | null; + mspa_covered_transactions?: boolean | null; + enable_tcfeu_string?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/GPPApplicationConfigResponse.ts b/clients/admin-ui/src/types/api/models/GPPApplicationConfigResponse.ts index d30eb9cb41..0905329ba0 100644 --- a/clients/admin-ui/src/types/api/models/GPPApplicationConfigResponse.ts +++ b/clients/admin-ui/src/types/api/models/GPPApplicationConfigResponse.ts @@ -8,10 +8,10 @@ import type { GPPUSApproach } from "./GPPUSApproach"; * Used to expose the _full_ GPP config in API responses */ export type GPPApplicationConfigResponse = { - us_approach?: GPPUSApproach; - mspa_service_provider_mode?: boolean; - mspa_opt_out_option_mode?: boolean; - mspa_covered_transactions?: boolean; - enable_tcfeu_string?: boolean; + us_approach?: GPPUSApproach | null; + mspa_service_provider_mode?: boolean | null; + mspa_opt_out_option_mode?: boolean | null; + mspa_covered_transactions?: boolean | null; + enable_tcfeu_string?: boolean | null; enabled: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/GPPFieldMapping.ts b/clients/admin-ui/src/types/api/models/GPPFieldMapping.ts index ba952f18db..fc2495b8f4 100644 --- a/clients/admin-ui/src/types/api/models/GPPFieldMapping.ts +++ b/clients/admin-ui/src/types/api/models/GPPFieldMapping.ts @@ -5,11 +5,8 @@ import type { GPPMechanismMapping } from "./GPPMechanismMapping"; import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type GPPFieldMapping = { region: PrivacyNoticeRegion; - notice?: Array; - mechanism?: Array; + notice?: Array | null; + mechanism?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/GPPFieldMappingCreate.ts b/clients/admin-ui/src/types/api/models/GPPFieldMappingCreate.ts index f19b5ebaab..20f5705f05 100644 --- a/clients/admin-ui/src/types/api/models/GPPFieldMappingCreate.ts +++ b/clients/admin-ui/src/types/api/models/GPPFieldMappingCreate.ts @@ -5,11 +5,8 @@ import type { GPPMechanismMapping } from "./GPPMechanismMapping"; import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type GPPFieldMappingCreate = { region: PrivacyNoticeRegion; - notice?: Array; - mechanism?: Array; + notice?: Array | null; + mechanism?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/GPPMechanismMapping.ts b/clients/admin-ui/src/types/api/models/GPPMechanismMapping.ts index fdae9b6139..c44c07f3d2 100644 --- a/clients/admin-ui/src/types/api/models/GPPMechanismMapping.ts +++ b/clients/admin-ui/src/types/api/models/GPPMechanismMapping.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type GPPMechanismMapping = { field: string; not_available: string; diff --git a/clients/admin-ui/src/types/api/models/GPPUSApproach.ts b/clients/admin-ui/src/types/api/models/GPPUSApproach.ts index cb42927522..fe2e01534a 100644 --- a/clients/admin-ui/src/types/api/models/GPPUSApproach.ts +++ b/clients/admin-ui/src/types/api/models/GPPUSApproach.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum GPPUSApproach { NATIONAL = "national", STATE = "state", diff --git a/clients/admin-ui/src/types/api/models/GenerateResponse.ts b/clients/admin-ui/src/types/api/models/GenerateResponse.ts index 8e230d7e77..0d9d9f0d65 100644 --- a/clients/admin-ui/src/types/api/models/GenerateResponse.ts +++ b/clients/admin-ui/src/types/api/models/GenerateResponse.ts @@ -2,12 +2,9 @@ /* tslint:disable */ /* eslint-disable */ -import type { Dataset } from "./Dataset"; -import type { System } from "./System"; - /** * The model to house the response for generated infrastructure. */ export type GenerateResponse = { - generate_results?: Array; + generate_results?: null; }; diff --git a/clients/admin-ui/src/types/api/models/GoogleCloudSQLPostgresDocsSchema.ts b/clients/admin-ui/src/types/api/models/GoogleCloudSQLPostgresDocsSchema.ts index 16a1d94e3e..682f4f5789 100644 --- a/clients/admin-ui/src/types/api/models/GoogleCloudSQLPostgresDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/GoogleCloudSQLPostgresDocsSchema.ts @@ -20,7 +20,7 @@ export type GoogleCloudSQLPostgresDocsSchema = { /** * The default schema to be used for the database connection (defaults to public). */ - db_schema?: string; + db_schema?: string | null; /** * The contents of the key file that contains authentication credentials for a service account in GCP. */ diff --git a/clients/admin-ui/src/types/api/models/Identity.ts b/clients/admin-ui/src/types/api/models/Identity.ts index 767199ebcf..b9e23b6e17 100644 --- a/clients/admin-ui/src/types/api/models/Identity.ts +++ b/clients/admin-ui/src/types/api/models/Identity.ts @@ -6,10 +6,10 @@ * Some PII grouping pertaining to a human */ export type Identity = { - phone_number?: string; - email?: string; - ga_client_id?: string; - ljt_readerID?: string; - fides_user_device_id?: string; - external_id?: string; + phone_number?: string | null; + email?: string | null; + ga_client_id?: string | null; + ljt_readerID?: string | null; + fides_user_device_id?: string | null; + external_id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/IdentityInputs.ts b/clients/admin-ui/src/types/api/models/IdentityInputs.ts index c9411685b4..fcd9a2e3d8 100644 --- a/clients/admin-ui/src/types/api/models/IdentityInputs.ts +++ b/clients/admin-ui/src/types/api/models/IdentityInputs.ts @@ -2,28 +2,8 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type IdentityInputs = { - name?: IdentityInputs.name; - email?: IdentityInputs.email; - phone?: IdentityInputs.phone; + name?: "optional" | "required" | null; + email?: "optional" | "required" | null; + phone?: "optional" | "required" | null; }; - -export namespace IdentityInputs { - export enum name { - OPTIONAL = "optional", - REQUIRED = "required", - } - - export enum email { - OPTIONAL = "optional", - REQUIRED = "required", - } - - export enum phone { - OPTIONAL = "optional", - REQUIRED = "required", - } -} diff --git a/clients/admin-ui/src/types/api/models/LimitedPrivacyNoticeResponseSchema.ts b/clients/admin-ui/src/types/api/models/LimitedPrivacyNoticeResponseSchema.ts index 9f7e0bea4a..a1816c6e6d 100644 --- a/clients/admin-ui/src/types/api/models/LimitedPrivacyNoticeResponseSchema.ts +++ b/clients/admin-ui/src/types/api/models/LimitedPrivacyNoticeResponseSchema.ts @@ -22,5 +22,5 @@ export type LimitedPrivacyNoticeResponseSchema = { configured_regions?: Array; systems_applicable?: boolean; disabled: boolean; - framework?: PrivacyNoticeFramework; + framework?: PrivacyNoticeFramework | null; }; diff --git a/clients/admin-ui/src/types/api/models/LocationRegulationResponse.ts b/clients/admin-ui/src/types/api/models/LocationRegulationResponse.ts index 9bd8bcd28f..c7543ada1f 100644 --- a/clients/admin-ui/src/types/api/models/LocationRegulationResponse.ts +++ b/clients/admin-ui/src/types/api/models/LocationRegulationResponse.ts @@ -6,9 +6,6 @@ import type { Location } from "./Location"; import type { LocationGroup } from "./LocationGroup"; import type { LocationRegulationBase } from "./LocationRegulationBase"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type LocationRegulationResponse = { locations?: Array; location_groups?: Array; diff --git a/clients/admin-ui/src/types/api/models/LocationRegulationSelections.ts b/clients/admin-ui/src/types/api/models/LocationRegulationSelections.ts index a2298ccd97..c1063973de 100644 --- a/clients/admin-ui/src/types/api/models/LocationRegulationSelections.ts +++ b/clients/admin-ui/src/types/api/models/LocationRegulationSelections.ts @@ -4,9 +4,6 @@ import type { Selection } from "./Selection"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type LocationRegulationSelections = { locations?: Array; regulations?: Array; diff --git a/clients/admin-ui/src/types/api/models/ManualAction.ts b/clients/admin-ui/src/types/api/models/ManualAction.ts index 7bb2781db6..3389294607 100644 --- a/clients/admin-ui/src/types/api/models/ManualAction.ts +++ b/clients/admin-ui/src/types/api/models/ManualAction.ts @@ -11,6 +11,6 @@ */ export type ManualAction = { locators: any; - get?: Array; - update?: any; + get: Array | null; + update: null; }; diff --git a/clients/admin-ui/src/types/api/models/ManualWebhookData.ts b/clients/admin-ui/src/types/api/models/ManualWebhookData.ts index bbc35831f5..6abd82856b 100644 --- a/clients/admin-ui/src/types/api/models/ManualWebhookData.ts +++ b/clients/admin-ui/src/types/api/models/ManualWebhookData.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type ManualWebhookData = { checked: boolean; fields: any; diff --git a/clients/admin-ui/src/types/api/models/ManualWebhookField.ts b/clients/admin-ui/src/types/api/models/ManualWebhookField.ts index 34610d0106..a4d41b5944 100644 --- a/clients/admin-ui/src/types/api/models/ManualWebhookField.ts +++ b/clients/admin-ui/src/types/api/models/ManualWebhookField.ts @@ -7,6 +7,6 @@ */ export type ManualWebhookField = { pii_field: string; - dsr_package_label?: string; - data_categories?: Array; + dsr_package_label?: string | null; + data_categories?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/MariaDBDocsSchema.ts b/clients/admin-ui/src/types/api/models/MariaDBDocsSchema.ts index 2decf30127..3569845f0c 100644 --- a/clients/admin-ui/src/types/api/models/MariaDBDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/MariaDBDocsSchema.ts @@ -17,11 +17,11 @@ export type MariaDBDocsSchema = { /** * The user account used to authenticate and access the database. */ - username?: string; + username?: string | null; /** * The password used to authenticate and access the database. */ - password?: string; + password?: string | null; /** * The name of the specific database within the database server that you want to connect to. */ diff --git a/clients/admin-ui/src/types/api/models/MessagingConfigRequest.ts b/clients/admin-ui/src/types/api/models/MessagingConfigRequest.ts index f42fbcb7a0..2c45d016ad 100644 --- a/clients/admin-ui/src/types/api/models/MessagingConfigRequest.ts +++ b/clients/admin-ui/src/types/api/models/MessagingConfigRequest.ts @@ -15,7 +15,8 @@ export type MessagingConfigRequest = { details?: | MessagingServiceDetailsMailgun | MessagingServiceDetailsTwilioEmail - | MessagingServiceDetailsMailchimpTransactional; + | MessagingServiceDetailsMailchimpTransactional + | null; name: string; - key?: string; + key?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/MessagingConfigRequestBase.ts b/clients/admin-ui/src/types/api/models/MessagingConfigRequestBase.ts index bbb7a6e64f..9a4eae5cf7 100644 --- a/clients/admin-ui/src/types/api/models/MessagingConfigRequestBase.ts +++ b/clients/admin-ui/src/types/api/models/MessagingConfigRequestBase.ts @@ -15,5 +15,6 @@ export type MessagingConfigRequestBase = { details?: | MessagingServiceDetailsMailgun | MessagingServiceDetailsTwilioEmail - | MessagingServiceDetailsMailchimpTransactional; + | MessagingServiceDetailsMailchimpTransactional + | null; }; diff --git a/clients/admin-ui/src/types/api/models/MessagingConfigResponse.ts b/clients/admin-ui/src/types/api/models/MessagingConfigResponse.ts index c806237d1c..036ed98d82 100644 --- a/clients/admin-ui/src/types/api/models/MessagingConfigResponse.ts +++ b/clients/admin-ui/src/types/api/models/MessagingConfigResponse.ts @@ -15,7 +15,8 @@ export type MessagingConfigResponse = { details?: | MessagingServiceDetailsMailgun | MessagingServiceDetailsTwilioEmail - | MessagingServiceDetailsMailchimpTransactional; + | MessagingServiceDetailsMailchimpTransactional + | null; name: string; key: string; }; diff --git a/clients/admin-ui/src/types/api/models/MessagingConfigStatusMessage.ts b/clients/admin-ui/src/types/api/models/MessagingConfigStatusMessage.ts index f8a2fd3b7b..16d4e8ac02 100644 --- a/clients/admin-ui/src/types/api/models/MessagingConfigStatusMessage.ts +++ b/clients/admin-ui/src/types/api/models/MessagingConfigStatusMessage.ts @@ -8,6 +8,6 @@ import type { MessagingConfigStatus } from "./MessagingConfigStatus"; * A schema for checking configuration status of message config. */ export type MessagingConfigStatusMessage = { - config_status?: MessagingConfigStatus; - detail?: string; + config_status?: MessagingConfigStatus | null; + detail?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/MessagingSecretsTwilioSMSDocs.ts b/clients/admin-ui/src/types/api/models/MessagingSecretsTwilioSMSDocs.ts index aea6cbe848..c9433cd8c2 100644 --- a/clients/admin-ui/src/types/api/models/MessagingSecretsTwilioSMSDocs.ts +++ b/clients/admin-ui/src/types/api/models/MessagingSecretsTwilioSMSDocs.ts @@ -8,6 +8,6 @@ export type MessagingSecretsTwilioSMSDocs = { twilio_account_sid: string; twilio_auth_token: string; - twilio_messaging_service_sid?: string; - twilio_sender_phone_number?: string; + twilio_messaging_service_sid?: string | null; + twilio_sender_phone_number?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/MessagingServiceDetailsMailgun.ts b/clients/admin-ui/src/types/api/models/MessagingServiceDetailsMailgun.ts index 510c54ab4a..cc1b031713 100644 --- a/clients/admin-ui/src/types/api/models/MessagingServiceDetailsMailgun.ts +++ b/clients/admin-ui/src/types/api/models/MessagingServiceDetailsMailgun.ts @@ -6,7 +6,7 @@ * The details required to represent a Mailgun email configuration. */ export type MessagingServiceDetailsMailgun = { - is_eu_domain?: boolean; - api_version?: string; + is_eu_domain?: boolean | null; + api_version?: string | null; domain: string; }; diff --git a/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesBodyParams.ts b/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesBodyParams.ts index 167365eb75..772a198284 100644 --- a/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesBodyParams.ts +++ b/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesBodyParams.ts @@ -4,6 +4,6 @@ export type MessagingTemplateWithPropertiesBodyParams = { content: any; - properties?: Array; + properties?: Array | null; is_enabled: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesDetail.ts b/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesDetail.ts index 75245319e1..55cfbbdd4b 100644 --- a/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesDetail.ts +++ b/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesDetail.ts @@ -8,6 +8,6 @@ export type MessagingTemplateWithPropertiesDetail = { id: string; type: string; is_enabled: boolean; - properties?: Array; + properties?: Array | null; content: any; }; diff --git a/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesPatchBodyParams.ts b/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesPatchBodyParams.ts index 7cfa624efb..ea1e172885 100644 --- a/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesPatchBodyParams.ts +++ b/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesPatchBodyParams.ts @@ -3,7 +3,7 @@ /* eslint-disable */ export type MessagingTemplateWithPropertiesPatchBodyParams = { - content?: any; - properties?: Array; - is_enabled?: boolean; + content?: null; + properties?: Array | null; + is_enabled?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesSummary.ts b/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesSummary.ts index 8d4284a802..a1b201db6e 100644 --- a/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesSummary.ts +++ b/clients/admin-ui/src/types/api/models/MessagingTemplateWithPropertiesSummary.ts @@ -8,5 +8,5 @@ export type MessagingTemplateWithPropertiesSummary = { id: string; type: string; is_enabled: boolean; - properties?: Array; + properties?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/MinimalProperty.ts b/clients/admin-ui/src/types/api/models/MinimalProperty.ts index f8abf03740..3408a59a48 100644 --- a/clients/admin-ui/src/types/api/models/MinimalProperty.ts +++ b/clients/admin-ui/src/types/api/models/MinimalProperty.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type MinimalProperty = { id: string; name: string; diff --git a/clients/admin-ui/src/types/api/models/MonitorClassifyParams.ts b/clients/admin-ui/src/types/api/models/MonitorClassifyParams.ts index 7a8d2eb354..6e109b5936 100644 --- a/clients/admin-ui/src/types/api/models/MonitorClassifyParams.ts +++ b/clients/admin-ui/src/types/api/models/MonitorClassifyParams.ts @@ -7,7 +7,7 @@ * classification parameters that can be used with Discovery monitors */ export type MonitorClassifyParams = { - possible_targets?: Array; + possible_targets?: null; top_n?: number; remove_stop_words?: boolean; pii_threshold?: number; @@ -21,5 +21,5 @@ export type MonitorClassifyParams = { decision_method?: string; aggregation_method?: string; infer_not_pii?: boolean; - context_regex_pattern_mapping?: Array>; + context_regex_pattern_mapping?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/MonitorConfig.ts b/clients/admin-ui/src/types/api/models/MonitorConfig.ts index 018bdc3ff9..cbd91f08ef 100644 --- a/clients/admin-ui/src/types/api/models/MonitorConfig.ts +++ b/clients/admin-ui/src/types/api/models/MonitorConfig.ts @@ -12,26 +12,26 @@ import type { S3MonitorParams } from "./S3MonitorParams"; */ export type MonitorConfig = { name: string; - key?: string; + key?: string | null; connection_config_key: string; classify_params: MonitorClassifyParams; /** * The datasource specific parameters, specified in a dictionary */ - datasource_params?: DynamoDBMonitorParams | S3MonitorParams; + datasource_params?: DynamoDBMonitorParams | S3MonitorParams | null; /** * The databases that the monitor is scoped to actively monitor */ databases?: Array; + execution_start_date?: string | null; + execution_frequency?: MonitorFrequency | null; /** * The databases that the monitor should exclude from monitoring */ excluded_databases?: Array; - execution_start_date?: string; - execution_frequency?: MonitorFrequency; /** * Indicates whether the monitor is enabled or not. Disabled monitors won't be executed */ - enabled?: boolean; - last_monitored?: string; + enabled?: boolean | null; + last_monitored?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/MonitorExecution.ts b/clients/admin-ui/src/types/api/models/MonitorExecution.ts index 6e1a0fadb0..23afec7193 100644 --- a/clients/admin-ui/src/types/api/models/MonitorExecution.ts +++ b/clients/admin-ui/src/types/api/models/MonitorExecution.ts @@ -7,8 +7,8 @@ import type { MonitorExecutionStatus } from "./MonitorExecutionStatus"; export type MonitorExecution = { id: string; monitor_config_id: string; - status?: MonitorExecutionStatus; - started?: string; - completed?: string; + status?: MonitorExecutionStatus | null; + started?: string | null; + completed?: string | null; classification_instances?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/MonitorExecutionStatus.ts b/clients/admin-ui/src/types/api/models/MonitorExecutionStatus.ts index 8d64431112..7cee4d550c 100644 --- a/clients/admin-ui/src/types/api/models/MonitorExecutionStatus.ts +++ b/clients/admin-ui/src/types/api/models/MonitorExecutionStatus.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum MonitorExecutionStatus { IN_PROGRESS = "In progress", COMPLETED = "Completed", diff --git a/clients/admin-ui/src/types/api/models/MySQLDocsSchema.ts b/clients/admin-ui/src/types/api/models/MySQLDocsSchema.ts index 52463034c6..88cd5ce94a 100644 --- a/clients/admin-ui/src/types/api/models/MySQLDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/MySQLDocsSchema.ts @@ -17,11 +17,11 @@ export type MySQLDocsSchema = { /** * The user account used to authenticate and access the database. */ - username?: string; + username?: string | null; /** * The password used to authenticate and access the database. */ - password?: string; + password?: string | null; /** * The name of the specific database within the database server that you want to connect to. */ diff --git a/clients/admin-ui/src/types/api/models/NoticeTranslation.ts b/clients/admin-ui/src/types/api/models/NoticeTranslation.ts index d6923fc2a5..38c98f0015 100644 --- a/clients/admin-ui/src/types/api/models/NoticeTranslation.ts +++ b/clients/admin-ui/src/types/api/models/NoticeTranslation.ts @@ -9,6 +9,6 @@ import type { SupportedLanguage } from "./SupportedLanguage"; */ export type NoticeTranslation = { language: SupportedLanguage; - title?: string; - description?: string; + title?: string | null; + description?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/NoticeTranslationCreate.ts b/clients/admin-ui/src/types/api/models/NoticeTranslationCreate.ts index 71b555c501..8aa3f99c28 100644 --- a/clients/admin-ui/src/types/api/models/NoticeTranslationCreate.ts +++ b/clients/admin-ui/src/types/api/models/NoticeTranslationCreate.ts @@ -11,5 +11,5 @@ import type { SupportedLanguage } from "./SupportedLanguage"; export type NoticeTranslationCreate = { language: SupportedLanguage; title: string; - description?: string; + description?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/NoticeTranslationResponse.ts b/clients/admin-ui/src/types/api/models/NoticeTranslationResponse.ts index 8586829fde..c6dbda7d3a 100644 --- a/clients/admin-ui/src/types/api/models/NoticeTranslationResponse.ts +++ b/clients/admin-ui/src/types/api/models/NoticeTranslationResponse.ts @@ -11,8 +11,8 @@ import type { SupportedLanguage } from "./SupportedLanguage"; */ export type NoticeTranslationResponse = { language: SupportedLanguage; - title?: string; - description?: string; + title?: string | null; + description?: string | null; /** * The versioned artifact of the translation and its Privacy Notice. Should be supplied when saving privacy preferences. */ diff --git a/clients/admin-ui/src/types/api/models/NotificationApplicationConfig.ts b/clients/admin-ui/src/types/api/models/NotificationApplicationConfig.ts index 41ef2a3fab..6f8df6b46b 100644 --- a/clients/admin-ui/src/types/api/models/NotificationApplicationConfig.ts +++ b/clients/admin-ui/src/types/api/models/NotificationApplicationConfig.ts @@ -6,9 +6,9 @@ * API model - configuration settings for data subject and/or data processor notifications */ export type NotificationApplicationConfig = { - send_request_completion_notification?: boolean; - send_request_receipt_notification?: boolean; - send_request_review_notification?: boolean; - notification_service_type?: string; - enable_property_specific_messaging?: boolean; + send_request_completion_notification?: boolean | null; + send_request_receipt_notification?: boolean | null; + send_request_review_notification?: boolean | null; + notification_service_type?: string | null; + enable_property_specific_messaging?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/OpenIDProvider.ts b/clients/admin-ui/src/types/api/models/OpenIDProvider.ts index 24075664b3..a83fd255c7 100644 --- a/clients/admin-ui/src/types/api/models/OpenIDProvider.ts +++ b/clients/admin-ui/src/types/api/models/OpenIDProvider.ts @@ -11,10 +11,10 @@ export type OpenIDProvider = { identifier: string; name: string; provider: ProviderEnum; - authorization_url?: string; - token_url?: string; - user_info_url?: string; - domain?: string; + authorization_url?: string | null; + token_url?: string | null; + user_info_url?: string | null; + domain?: string | null; id: string; created_at: string; updated_at: string; diff --git a/clients/admin-ui/src/types/api/models/OpenIDProviderCreate.ts b/clients/admin-ui/src/types/api/models/OpenIDProviderCreate.ts index b4466a5e2a..d0ece0429e 100644 --- a/clients/admin-ui/src/types/api/models/OpenIDProviderCreate.ts +++ b/clients/admin-ui/src/types/api/models/OpenIDProviderCreate.ts @@ -11,10 +11,10 @@ export type OpenIDProviderCreate = { identifier: string; name: string; provider: ProviderEnum; - authorization_url?: string; - token_url?: string; - user_info_url?: string; - domain?: string; + authorization_url?: string | null; + token_url?: string | null; + user_info_url?: string | null; + domain?: string | null; client_id: string; client_secret: string; }; diff --git a/clients/admin-ui/src/types/api/models/Organization.ts b/clients/admin-ui/src/types/api/models/Organization.ts index f0363e8f72..8b46e1d320 100644 --- a/clients/admin-ui/src/types/api/models/Organization.ts +++ b/clients/admin-ui/src/types/api/models/Organization.ts @@ -19,15 +19,15 @@ export type Organization = { * Defines the Organization that this resource belongs to. */ organization_fides_key?: string; - tags?: Array; + tags?: Array | null; /** * Human-Readable name for this resource. */ - name?: string; + name?: string | null; /** * A detailed description of what this resource is. */ - description?: string; + description?: string | null; /** * An inherited field from the FidesModel that is unused with an Organization. */ @@ -43,7 +43,7 @@ export type Organization = { * potentially under a system/dataset. * */ - controller?: ContactDetails; + controller?: ContactDetails | null; /** * * The contact details information model. @@ -55,7 +55,7 @@ export type Organization = { * potentially under a system/dataset. * */ - data_protection_officer?: ContactDetails; + data_protection_officer?: ContactDetails | null; /** * * The OrganizationMetadata resource model. @@ -63,7 +63,7 @@ export type Organization = { * Object used to hold application specific metadata for an organization * */ - fidesctl_meta?: OrganizationMetadata; + fidesctl_meta?: OrganizationMetadata | null; /** * * The contact details information model. @@ -75,9 +75,9 @@ export type Organization = { * potentially under a system/dataset. * */ - representative?: ContactDetails; + representative?: ContactDetails | null; /** * Am optional URL to the organization security policy. */ - security_policy?: string; + security_policy?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/OrganizationMetadata.ts b/clients/admin-ui/src/types/api/models/OrganizationMetadata.ts index 2593cde619..6168d114c6 100644 --- a/clients/admin-ui/src/types/api/models/OrganizationMetadata.ts +++ b/clients/admin-ui/src/types/api/models/OrganizationMetadata.ts @@ -13,5 +13,5 @@ export type OrganizationMetadata = { /** * A list of filters that can be used when generating or scanning systems. */ - resource_filters?: Array; + resource_filters?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_BasicSystemResponse_.ts b/clients/admin-ui/src/types/api/models/Page_BasicSystemResponse_.ts index 8a8ef4ec75..749a424cc2 100644 --- a/clients/admin-ui/src/types/api/models/Page_BasicSystemResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_BasicSystemResponse_.ts @@ -6,8 +6,8 @@ import type { BasicSystemResponse } from "./BasicSystemResponse"; export type Page_BasicSystemResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_ClassifyInstanceResponseValues_.ts b/clients/admin-ui/src/types/api/models/Page_ClassifyInstanceResponseValues_.ts index d7b16fdbc1..a1e0b0caad 100644 --- a/clients/admin-ui/src/types/api/models/Page_ClassifyInstanceResponseValues_.ts +++ b/clients/admin-ui/src/types/api/models/Page_ClassifyInstanceResponseValues_.ts @@ -6,8 +6,8 @@ import type { ClassifyInstanceResponseValues } from "./ClassifyInstanceResponseV export type Page_ClassifyInstanceResponseValues_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_ConnectionConfigurationResponse_.ts b/clients/admin-ui/src/types/api/models/Page_ConnectionConfigurationResponse_.ts index 485d6a5867..9c48b66d48 100644 --- a/clients/admin-ui/src/types/api/models/Page_ConnectionConfigurationResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_ConnectionConfigurationResponse_.ts @@ -6,8 +6,8 @@ import type { ConnectionConfigurationResponse } from "./ConnectionConfigurationR export type Page_ConnectionConfigurationResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_ConnectionSystemTypeMap_.ts b/clients/admin-ui/src/types/api/models/Page_ConnectionSystemTypeMap_.ts index 3fba9300fc..7ac0b84482 100644 --- a/clients/admin-ui/src/types/api/models/Page_ConnectionSystemTypeMap_.ts +++ b/clients/admin-ui/src/types/api/models/Page_ConnectionSystemTypeMap_.ts @@ -6,8 +6,8 @@ import type { ConnectionSystemTypeMap } from "./ConnectionSystemTypeMap"; export type Page_ConnectionSystemTypeMap_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_ConsentReport_.ts b/clients/admin-ui/src/types/api/models/Page_ConsentReport_.ts index 90fbf5b771..251ca7badc 100644 --- a/clients/admin-ui/src/types/api/models/Page_ConsentReport_.ts +++ b/clients/admin-ui/src/types/api/models/Page_ConsentReport_.ts @@ -6,8 +6,8 @@ import type { ConsentReport } from "./ConsentReport"; export type Page_ConsentReport_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_ConsentReportingSchema_.ts b/clients/admin-ui/src/types/api/models/Page_ConsentReportingSchema_.ts index 363256e6ad..3d58689638 100644 --- a/clients/admin-ui/src/types/api/models/Page_ConsentReportingSchema_.ts +++ b/clients/admin-ui/src/types/api/models/Page_ConsentReportingSchema_.ts @@ -6,8 +6,8 @@ import type { ConsentReportingSchema } from "./ConsentReportingSchema"; export type Page_ConsentReportingSchema_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_DatamapReport_.ts b/clients/admin-ui/src/types/api/models/Page_DatamapReport_.ts index e327d02ff8..bdff5edbe3 100644 --- a/clients/admin-ui/src/types/api/models/Page_DatamapReport_.ts +++ b/clients/admin-ui/src/types/api/models/Page_DatamapReport_.ts @@ -6,8 +6,8 @@ import type { DatamapReport } from "./DatamapReport"; export type Page_DatamapReport_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_DatasetConfigSchema_.ts b/clients/admin-ui/src/types/api/models/Page_DatasetConfigSchema_.ts index 9f56b71df1..23a2e34e01 100644 --- a/clients/admin-ui/src/types/api/models/Page_DatasetConfigSchema_.ts +++ b/clients/admin-ui/src/types/api/models/Page_DatasetConfigSchema_.ts @@ -6,8 +6,8 @@ import type { DatasetConfigSchema } from "./DatasetConfigSchema"; export type Page_DatasetConfigSchema_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_Dataset_.ts b/clients/admin-ui/src/types/api/models/Page_Dataset_.ts index f7b22d7980..a707f22ace 100644 --- a/clients/admin-ui/src/types/api/models/Page_Dataset_.ts +++ b/clients/admin-ui/src/types/api/models/Page_Dataset_.ts @@ -6,8 +6,8 @@ import type { Dataset } from "./Dataset"; export type Page_Dataset_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_ExecutionLogDetailResponse_.ts b/clients/admin-ui/src/types/api/models/Page_ExecutionLogDetailResponse_.ts index e7b209a2b7..6cefd0161f 100644 --- a/clients/admin-ui/src/types/api/models/Page_ExecutionLogDetailResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_ExecutionLogDetailResponse_.ts @@ -6,8 +6,8 @@ import type { ExecutionLogDetailResponse } from "./ExecutionLogDetailResponse"; export type Page_ExecutionLogDetailResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_ExperienceConfigListViewResponse_.ts b/clients/admin-ui/src/types/api/models/Page_ExperienceConfigListViewResponse_.ts index 30e14de278..a0b11c441d 100644 --- a/clients/admin-ui/src/types/api/models/Page_ExperienceConfigListViewResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_ExperienceConfigListViewResponse_.ts @@ -6,8 +6,8 @@ import type { ExperienceConfigListViewResponse } from "./ExperienceConfigListVie export type Page_ExperienceConfigListViewResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_Language_.ts b/clients/admin-ui/src/types/api/models/Page_Language_.ts index 7db5aa5675..26b528acc5 100644 --- a/clients/admin-ui/src/types/api/models/Page_Language_.ts +++ b/clients/admin-ui/src/types/api/models/Page_Language_.ts @@ -6,8 +6,8 @@ import type { Language } from "./Language"; export type Page_Language_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_LimitedPrivacyNoticeResponseSchema_.ts b/clients/admin-ui/src/types/api/models/Page_LimitedPrivacyNoticeResponseSchema_.ts index 8e60db8ec5..4b181a4a4e 100644 --- a/clients/admin-ui/src/types/api/models/Page_LimitedPrivacyNoticeResponseSchema_.ts +++ b/clients/admin-ui/src/types/api/models/Page_LimitedPrivacyNoticeResponseSchema_.ts @@ -6,8 +6,8 @@ import type { LimitedPrivacyNoticeResponseSchema } from "./LimitedPrivacyNoticeR export type Page_LimitedPrivacyNoticeResponseSchema_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_MessagingConfigResponse_.ts b/clients/admin-ui/src/types/api/models/Page_MessagingConfigResponse_.ts index 857f0f9c7c..5e28aa20ae 100644 --- a/clients/admin-ui/src/types/api/models/Page_MessagingConfigResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_MessagingConfigResponse_.ts @@ -6,8 +6,8 @@ import type { MessagingConfigResponse } from "./MessagingConfigResponse"; export type Page_MessagingConfigResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_MessagingTemplateWithPropertiesSummary_.ts b/clients/admin-ui/src/types/api/models/Page_MessagingTemplateWithPropertiesSummary_.ts index fce6ad50d1..97cd85967f 100644 --- a/clients/admin-ui/src/types/api/models/Page_MessagingTemplateWithPropertiesSummary_.ts +++ b/clients/admin-ui/src/types/api/models/Page_MessagingTemplateWithPropertiesSummary_.ts @@ -6,8 +6,8 @@ import type { MessagingTemplateWithPropertiesSummary } from "./MessagingTemplate export type Page_MessagingTemplateWithPropertiesSummary_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_MonitorConfig_.ts b/clients/admin-ui/src/types/api/models/Page_MonitorConfig_.ts index 43fd0f327f..3c877a6947 100644 --- a/clients/admin-ui/src/types/api/models/Page_MonitorConfig_.ts +++ b/clients/admin-ui/src/types/api/models/Page_MonitorConfig_.ts @@ -6,8 +6,8 @@ import type { MonitorConfig } from "./MonitorConfig"; export type Page_MonitorConfig_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_MonitorExecution_.ts b/clients/admin-ui/src/types/api/models/Page_MonitorExecution_.ts index 56842aadba..186a68953d 100644 --- a/clients/admin-ui/src/types/api/models/Page_MonitorExecution_.ts +++ b/clients/admin-ui/src/types/api/models/Page_MonitorExecution_.ts @@ -6,8 +6,8 @@ import type { MonitorExecution } from "./MonitorExecution"; export type Page_MonitorExecution_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_PolicyResponse_.ts b/clients/admin-ui/src/types/api/models/Page_PolicyResponse_.ts index 7ec6f9543f..81f3609d0a 100644 --- a/clients/admin-ui/src/types/api/models/Page_PolicyResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_PolicyResponse_.ts @@ -6,8 +6,8 @@ import type { PolicyResponse } from "./PolicyResponse"; export type Page_PolicyResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_PolicyWebhookResponse_.ts b/clients/admin-ui/src/types/api/models/Page_PolicyWebhookResponse_.ts index 8a446e189e..52f3ecf964 100644 --- a/clients/admin-ui/src/types/api/models/Page_PolicyWebhookResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_PolicyWebhookResponse_.ts @@ -6,8 +6,8 @@ import type { PolicyWebhookResponse } from "./PolicyWebhookResponse"; export type Page_PolicyWebhookResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_PreApprovalWebhookResponse_.ts b/clients/admin-ui/src/types/api/models/Page_PreApprovalWebhookResponse_.ts index ed96b9b18a..294bfc3c26 100644 --- a/clients/admin-ui/src/types/api/models/Page_PreApprovalWebhookResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_PreApprovalWebhookResponse_.ts @@ -6,8 +6,8 @@ import type { PreApprovalWebhookResponse } from "./PreApprovalWebhookResponse"; export type Page_PreApprovalWebhookResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_PrivacyExperienceMetaResponse_.ts b/clients/admin-ui/src/types/api/models/Page_PrivacyExperienceMetaResponse_.ts index f6fc715522..0d24acd707 100644 --- a/clients/admin-ui/src/types/api/models/Page_PrivacyExperienceMetaResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_PrivacyExperienceMetaResponse_.ts @@ -6,8 +6,8 @@ import type { PrivacyExperienceMetaResponse } from "./PrivacyExperienceMetaRespo export type Page_PrivacyExperienceMetaResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_PrivacyExperienceResponse_.ts b/clients/admin-ui/src/types/api/models/Page_PrivacyExperienceResponse_.ts index 4baf87a33c..0c2b88e3db 100644 --- a/clients/admin-ui/src/types/api/models/Page_PrivacyExperienceResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_PrivacyExperienceResponse_.ts @@ -6,8 +6,8 @@ import type { PrivacyExperienceResponse } from "./PrivacyExperienceResponse"; export type Page_PrivacyExperienceResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_Property_.ts b/clients/admin-ui/src/types/api/models/Page_Property_.ts index 0165814dfe..df6df75e2f 100644 --- a/clients/admin-ui/src/types/api/models/Page_Property_.ts +++ b/clients/admin-ui/src/types/api/models/Page_Property_.ts @@ -6,8 +6,8 @@ import type { Property } from "./Property"; export type Page_Property_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_RuleResponseWithTargets_.ts b/clients/admin-ui/src/types/api/models/Page_RuleResponseWithTargets_.ts index 944528d8cd..992cd61b77 100644 --- a/clients/admin-ui/src/types/api/models/Page_RuleResponseWithTargets_.ts +++ b/clients/admin-ui/src/types/api/models/Page_RuleResponseWithTargets_.ts @@ -6,8 +6,8 @@ import type { RuleResponseWithTargets } from "./RuleResponseWithTargets"; export type Page_RuleResponseWithTargets_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_RuleTarget_.ts b/clients/admin-ui/src/types/api/models/Page_RuleTarget_.ts index 8e7fa84145..e53db526ec 100644 --- a/clients/admin-ui/src/types/api/models/Page_RuleTarget_.ts +++ b/clients/admin-ui/src/types/api/models/Page_RuleTarget_.ts @@ -6,8 +6,8 @@ import type { RuleTarget } from "./RuleTarget"; export type Page_RuleTarget_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_StagedResourceAPIResponse_.ts b/clients/admin-ui/src/types/api/models/Page_StagedResourceAPIResponse_.ts index df7aea9d9b..0c164900b9 100644 --- a/clients/admin-ui/src/types/api/models/Page_StagedResourceAPIResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_StagedResourceAPIResponse_.ts @@ -6,8 +6,8 @@ import type { StagedResourceAPIResponse } from "./StagedResourceAPIResponse"; export type Page_StagedResourceAPIResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_StorageDestinationResponse_.ts b/clients/admin-ui/src/types/api/models/Page_StorageDestinationResponse_.ts index 6843a70360..4692691cd6 100644 --- a/clients/admin-ui/src/types/api/models/Page_StorageDestinationResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_StorageDestinationResponse_.ts @@ -6,8 +6,8 @@ import type { StorageDestinationResponse } from "./StorageDestinationResponse"; export type Page_StorageDestinationResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_SystemHistoryResponse_.ts b/clients/admin-ui/src/types/api/models/Page_SystemHistoryResponse_.ts index 9550b01a0b..163d78e5d8 100644 --- a/clients/admin-ui/src/types/api/models/Page_SystemHistoryResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_SystemHistoryResponse_.ts @@ -6,8 +6,8 @@ import type { SystemHistoryResponse } from "./SystemHistoryResponse"; export type Page_SystemHistoryResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_SystemSummary_.ts b/clients/admin-ui/src/types/api/models/Page_SystemSummary_.ts index d23e9b4386..951449f678 100644 --- a/clients/admin-ui/src/types/api/models/Page_SystemSummary_.ts +++ b/clients/admin-ui/src/types/api/models/Page_SystemSummary_.ts @@ -6,8 +6,8 @@ import type { SystemSummary } from "./SystemSummary"; export type Page_SystemSummary_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_Union_PrivacyRequestVerboseResponse__PrivacyRequestResponse__.ts b/clients/admin-ui/src/types/api/models/Page_Union_PrivacyRequestVerboseResponse__PrivacyRequestResponse__.ts index 88c9c45c66..b3c72df7ee 100644 --- a/clients/admin-ui/src/types/api/models/Page_Union_PrivacyRequestVerboseResponse__PrivacyRequestResponse__.ts +++ b/clients/admin-ui/src/types/api/models/Page_Union_PrivacyRequestVerboseResponse__PrivacyRequestResponse__.ts @@ -8,8 +8,8 @@ import type { PrivacyRequestVerboseResponse } from "./PrivacyRequestVerboseRespo export type Page_Union_PrivacyRequestVerboseResponse__PrivacyRequestResponse__ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_UserResponse_.ts b/clients/admin-ui/src/types/api/models/Page_UserResponse_.ts index e844ab5735..518509b901 100644 --- a/clients/admin-ui/src/types/api/models/Page_UserResponse_.ts +++ b/clients/admin-ui/src/types/api/models/Page_UserResponse_.ts @@ -6,8 +6,8 @@ import type { UserResponse } from "./UserResponse"; export type Page_UserResponse_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/Page_str_.ts b/clients/admin-ui/src/types/api/models/Page_str_.ts index 62efc93e65..cdbce2522b 100644 --- a/clients/admin-ui/src/types/api/models/Page_str_.ts +++ b/clients/admin-ui/src/types/api/models/Page_str_.ts @@ -4,8 +4,8 @@ export type Page_str_ = { items: Array; - total: number; - page: number; - size: number; - pages?: number; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/ParamValue.ts b/clients/admin-ui/src/types/api/models/ParamValue.ts index b8d85128e6..c6ce40fd19 100644 --- a/clients/admin-ui/src/types/api/models/ParamValue.ts +++ b/clients/admin-ui/src/types/api/models/ParamValue.ts @@ -2,16 +2,14 @@ /* tslint:disable */ /* eslint-disable */ -import type { FidesDatasetReference } from "./FidesDatasetReference"; - /** * A named variable which can be sourced from identities, dataset references, or connector params. These values * are used to replace the placeholders in the path, header, query, and body param values. */ export type ParamValue = { name: string; - identity?: string; - references?: Array; - connector_param?: string; - unpack?: boolean; + identity?: string | null; + references?: null; + connector_param?: string | null; + unpack?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/PartialPrivacyRequestOption.ts b/clients/admin-ui/src/types/api/models/PartialPrivacyRequestOption.ts index f10f9ae8ad..605d5d3152 100644 --- a/clients/admin-ui/src/types/api/models/PartialPrivacyRequestOption.ts +++ b/clients/admin-ui/src/types/api/models/PartialPrivacyRequestOption.ts @@ -5,15 +5,12 @@ import type { fides__api__schemas__privacy_center_config__CustomPrivacyRequestField } from "./fides__api__schemas__privacy_center_config__CustomPrivacyRequestField"; import type { IdentityInputs } from "./IdentityInputs"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type PartialPrivacyRequestOption = { policy_key: string; title: string; - identity_inputs?: IdentityInputs; + identity_inputs?: IdentityInputs | null; custom_privacy_request_fields?: Record< string, fides__api__schemas__privacy_center_config__CustomPrivacyRequestField - >; + > | null; }; diff --git a/clients/admin-ui/src/types/api/models/PlusApplicationConfig.ts b/clients/admin-ui/src/types/api/models/PlusApplicationConfig.ts index 47821a8c4a..fcfa9ad2a4 100644 --- a/clients/admin-ui/src/types/api/models/PlusApplicationConfig.ts +++ b/clients/admin-ui/src/types/api/models/PlusApplicationConfig.ts @@ -10,19 +10,12 @@ import type { NotificationApplicationConfig } from "./NotificationApplicationCon import type { SecurityApplicationConfig } from "./SecurityApplicationConfig"; import type { StorageApplicationConfig } from "./StorageApplicationConfig"; -/** - * Application config settings update body is an arbitrary dict (JSON object) - * We describe it in a schema to enforce some restrictions on the keys passed. - * - * TODO: Eventually this should be driven by a more formal validation schema for this - * the application config that is properly hooked up to the global pydantic config module. - */ export type PlusApplicationConfig = { - storage?: StorageApplicationConfig; - notifications?: NotificationApplicationConfig; - execution?: ExecutionApplicationConfig; - security?: SecurityApplicationConfig; - consent?: fides__api__schemas__application_config__ConsentConfig; - admin_ui?: AdminUIConfig; - gpp?: GPPApplicationConfig; + storage?: StorageApplicationConfig | null; + notifications?: NotificationApplicationConfig | null; + execution?: ExecutionApplicationConfig | null; + security?: SecurityApplicationConfig | null; + consent?: fides__api__schemas__application_config__ConsentConfig | null; + admin_ui?: AdminUIConfig | null; + gpp?: GPPApplicationConfig | null; }; diff --git a/clients/admin-ui/src/types/api/models/PolicyResponse.ts b/clients/admin-ui/src/types/api/models/PolicyResponse.ts index b0e474557f..8ac37a4d51 100644 --- a/clients/admin-ui/src/types/api/models/PolicyResponse.ts +++ b/clients/admin-ui/src/types/api/models/PolicyResponse.ts @@ -10,8 +10,8 @@ import type { RuleResponse } from "./RuleResponse"; */ export type PolicyResponse = { name: string; - key?: string; - drp_action?: DrpAction; - execution_timeframe?: number; - rules?: Array; + key?: string | null; + drp_action?: DrpAction | null; + execution_timeframe?: number | null; + rules?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/PolicyWebhookCreate.ts b/clients/admin-ui/src/types/api/models/PolicyWebhookCreate.ts index c67c7e13cb..f5037e256d 100644 --- a/clients/admin-ui/src/types/api/models/PolicyWebhookCreate.ts +++ b/clients/admin-ui/src/types/api/models/PolicyWebhookCreate.ts @@ -9,7 +9,7 @@ import type { WebhookDirection } from "./WebhookDirection"; */ export type PolicyWebhookCreate = { direction: WebhookDirection; - key?: string; - name?: string; + key?: string | null; + name?: string | null; connection_config_key: string; }; diff --git a/clients/admin-ui/src/types/api/models/PolicyWebhookResponse.ts b/clients/admin-ui/src/types/api/models/PolicyWebhookResponse.ts index 0b24051b20..17c6d4a475 100644 --- a/clients/admin-ui/src/types/api/models/PolicyWebhookResponse.ts +++ b/clients/admin-ui/src/types/api/models/PolicyWebhookResponse.ts @@ -10,8 +10,8 @@ import type { WebhookDirection } from "./WebhookDirection"; */ export type PolicyWebhookResponse = { direction: WebhookDirection; - key?: string; - name?: string; - connection_config?: ConnectionConfigurationResponse; + key?: string | null; + name?: string | null; + connection_config?: ConnectionConfigurationResponse | null; order: number; }; diff --git a/clients/admin-ui/src/types/api/models/PolicyWebhookUpdate.ts b/clients/admin-ui/src/types/api/models/PolicyWebhookUpdate.ts index c4ad43f573..630ff9949c 100644 --- a/clients/admin-ui/src/types/api/models/PolicyWebhookUpdate.ts +++ b/clients/admin-ui/src/types/api/models/PolicyWebhookUpdate.ts @@ -8,8 +8,8 @@ import type { WebhookDirection } from "./WebhookDirection"; * Request schema for updating a single webhook - fields are optional */ export type PolicyWebhookUpdate = { - direction?: WebhookDirection; - name?: string; - connection_config_key?: string; - order?: number; + direction?: WebhookDirection | null; + name?: string | null; + connection_config_key?: string | null; + order?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts b/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts index be0c66afc9..801d10c24a 100644 --- a/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts @@ -17,11 +17,11 @@ export type PostgreSQLDocsSchema = { /** * The user account used to authenticate and access the database. */ - username?: string; + username?: string | null; /** * The password used to authenticate and access the database. */ - password?: string; + password?: string | null; /** * The name of the specific database within the database server that you want to connect to. */ @@ -29,7 +29,7 @@ export type PostgreSQLDocsSchema = { /** * The default schema to be used for the database connection (defaults to public). */ - db_schema?: string; + db_schema?: string | null; /** * Indicates whether an SSH tunnel is required for the connection. Enable this option if your PostgreSQL server is behind a firewall and requires SSH tunneling for remote connections. */ diff --git a/clients/admin-ui/src/types/api/models/PreApprovalWebhookCreate.ts b/clients/admin-ui/src/types/api/models/PreApprovalWebhookCreate.ts index 2bef7a12db..bec8444f59 100644 --- a/clients/admin-ui/src/types/api/models/PreApprovalWebhookCreate.ts +++ b/clients/admin-ui/src/types/api/models/PreApprovalWebhookCreate.ts @@ -6,7 +6,7 @@ * Request schema for creating/updating a Pre Approval Webhook */ export type PreApprovalWebhookCreate = { - key?: string; + key?: string | null; name: string; connection_config_key: string; }; diff --git a/clients/admin-ui/src/types/api/models/PreApprovalWebhookResponse.ts b/clients/admin-ui/src/types/api/models/PreApprovalWebhookResponse.ts index eaea50a176..4991aa7420 100644 --- a/clients/admin-ui/src/types/api/models/PreApprovalWebhookResponse.ts +++ b/clients/admin-ui/src/types/api/models/PreApprovalWebhookResponse.ts @@ -8,7 +8,7 @@ import type { ConnectionConfigurationResponse } from "./ConnectionConfigurationR * Response schema after creating/updating/getting a PreApprovalWebhook */ export type PreApprovalWebhookResponse = { - key?: string; + key?: string | null; name: string; - connection_config?: ConnectionConfigurationResponse; + connection_config?: ConnectionConfigurationResponse | null; }; diff --git a/clients/admin-ui/src/types/api/models/PreApprovalWebhookUpdate.ts b/clients/admin-ui/src/types/api/models/PreApprovalWebhookUpdate.ts index e01c198ad9..43af9f3b36 100644 --- a/clients/admin-ui/src/types/api/models/PreApprovalWebhookUpdate.ts +++ b/clients/admin-ui/src/types/api/models/PreApprovalWebhookUpdate.ts @@ -6,6 +6,6 @@ * Request schema for updating a single webhook - fields are optional */ export type PreApprovalWebhookUpdate = { - name?: string; - connection_config_key?: string; + name?: string | null; + connection_config_key?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyCenterConfig.ts b/clients/admin-ui/src/types/api/models/PrivacyCenterConfig.ts index a8de3e5487..61cc2a06a9 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyCenterConfig.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyCenterConfig.ts @@ -14,16 +14,16 @@ import type { PrivacyRequestOption } from "./PrivacyRequestOption"; export type PrivacyCenterConfig = { title: string; description: string; - description_subtext?: Array; - addendum?: Array; - server_url_development?: string; - server_url_production?: string; - logo_path?: string; - logo_url?: string; - favicon_path?: string; + description_subtext?: Array | null; + addendum?: Array | null; + server_url_development?: string | null; + server_url_production?: string | null; + logo_path?: string | null; + logo_url?: string | null; + favicon_path?: string | null; actions: Array; - includeConsent?: boolean; + includeConsent?: boolean | null; consent: fides__api__schemas__privacy_center_config__ConsentConfig; - privacy_policy_url?: string; - privacy_policy_url_text?: string; + privacy_policy_url?: string | null; + privacy_policy_url_text?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts b/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts index f46e150c08..f51b8dbc1c 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts @@ -16,7 +16,7 @@ export type PrivacyDeclaration = { /** * The name of the privacy declaration on the system. */ - name?: string; + name?: string | null; /** * An array of data categories describing a system in a privacy declaration. */ @@ -32,15 +32,15 @@ export type PrivacyDeclaration = { /** * Referenced Dataset fides keys used by the system. */ - dataset_references?: Array; + dataset_references?: Array | null; /** * The resources to which data is sent. Any `fides_key`s included in this list reference `DataFlow` entries in the `egress` array of any `System` resources to which this `PrivacyDeclaration` is applied. */ - egress?: Array; + egress?: Array | null; /** * The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied. */ - ingress?: Array; + ingress?: Array | null; /** * The features of processing personal data. */ @@ -52,15 +52,15 @@ export type PrivacyDeclaration = { /** * The legal basis under which personal data is processed for this purpose. */ - legal_basis_for_processing?: LegalBasisForProcessingEnum; + legal_basis_for_processing?: LegalBasisForProcessingEnum | null; /** * Where the legitimate interest impact assessment is stored */ - impact_assessment_location?: string; + impact_assessment_location?: string | null; /** * An optional string to describe the time period for which data is retained for this purpose. */ - retention_period?: string; + retention_period?: string | null; /** * This system processes special category data */ @@ -68,7 +68,7 @@ export type PrivacyDeclaration = { /** * The legal basis under which the special category data is processed. */ - special_category_legal_basis?: SpecialCategoryLegalBasisEnum; + special_category_legal_basis?: SpecialCategoryLegalBasisEnum | null; /** * This system shares data with third parties for this purpose. */ @@ -76,7 +76,7 @@ export type PrivacyDeclaration = { /** * The types of third parties the data is shared with. */ - third_parties?: string; + third_parties?: string | null; /** * The categories of personal data that this system shares with third parties. */ @@ -84,5 +84,5 @@ export type PrivacyDeclaration = { /** * Cookies associated with this data use to deliver services and functionality */ - cookies?: Array; + cookies?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts index 4cfc6ee11f..9878bfb6db 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts @@ -13,7 +13,7 @@ export type PrivacyDeclarationResponse = { /** * The name of the privacy declaration on the system. */ - name?: string; + name?: string | null; /** * An array of data categories describing a system in a privacy declaration. */ @@ -29,15 +29,15 @@ export type PrivacyDeclarationResponse = { /** * Referenced Dataset fides keys used by the system. */ - dataset_references?: Array; + dataset_references?: Array | null; /** * The resources to which data is sent. Any `fides_key`s included in this list reference `DataFlow` entries in the `egress` array of any `System` resources to which this `PrivacyDeclaration` is applied. */ - egress?: Array; + egress?: Array | null; /** * The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied. */ - ingress?: Array; + ingress?: Array | null; /** * The features of processing personal data. */ @@ -49,15 +49,15 @@ export type PrivacyDeclarationResponse = { /** * The legal basis under which personal data is processed for this purpose. */ - legal_basis_for_processing?: LegalBasisForProcessingEnum; + legal_basis_for_processing?: LegalBasisForProcessingEnum | null; /** * Where the legitimate interest impact assessment is stored */ - impact_assessment_location?: string; + impact_assessment_location?: string | null; /** * An optional string to describe the time period for which data is retained for this purpose. */ - retention_period?: string; + retention_period?: string | null; /** * This system processes special category data */ @@ -65,7 +65,7 @@ export type PrivacyDeclarationResponse = { /** * The legal basis under which the special category data is processed. */ - special_category_legal_basis?: SpecialCategoryLegalBasisEnum; + special_category_legal_basis?: SpecialCategoryLegalBasisEnum | null; /** * This system shares data with third parties for this purpose. */ @@ -73,12 +73,12 @@ export type PrivacyDeclarationResponse = { /** * The types of third parties the data is shared with. */ - third_parties?: string; + third_parties?: string | null; /** * The categories of personal data that this system shares with third parties. */ shared_categories?: Array; - cookies?: Array; + cookies?: Array | null; /** * The database-assigned ID of the privacy declaration on the system. This is meant to be a read-only field, returned only in API responses */ diff --git a/clients/admin-ui/src/types/api/models/PrivacyExperienceMetaResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyExperienceMetaResponse.ts index 7ee91c80b0..c435069c72 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyExperienceMetaResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyExperienceMetaResponse.ts @@ -12,6 +12,6 @@ import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; export type PrivacyExperienceMetaResponse = { id: string; region: PrivacyNoticeRegion; - component?: ComponentType; - meta?: ExperienceMeta; + component?: ComponentType | null; + meta?: ExperienceMeta | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyExperienceResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyExperienceResponse.ts index 507319f409..0699d09a8d 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyExperienceResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyExperienceResponse.ts @@ -31,8 +31,8 @@ export type PrivacyExperienceResponse = { created_at: string; updated_at: string; region: PrivacyNoticeRegion; - component?: ComponentType; - gpp_settings?: GPPApplicationConfigResponse; + component?: ComponentType | null; + gpp_settings?: GPPApplicationConfigResponse | null; tcf_purpose_consents?: Array; tcf_purpose_legitimate_interests?: Array; tcf_special_purposes?: Array; @@ -47,17 +47,17 @@ export type PrivacyExperienceResponse = { /** * For backwards compatibility purposes, whether the Experience should show a banner. */ - show_banner?: boolean; + show_banner?: boolean | null; /** * The Privacy Notices associated with this experience, if applicable */ - privacy_notices?: Array; + privacy_notices?: Array | null; /** * The Experience Config and its translations */ - experience_config?: ExperienceConfigResponseNoNotices; - gvl?: any; - gvl_translations?: any; - available_locales?: Array; - meta?: ExperienceMeta; + experience_config?: ExperienceConfigResponseNoNotices | null; + gvl?: null; + gvl_translations?: null; + available_locales?: Array | null; + meta?: ExperienceMeta | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyNoticeCreation.ts b/clients/admin-ui/src/types/api/models/PrivacyNoticeCreation.ts index eaff237999..d7e1014c68 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyNoticeCreation.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyNoticeCreation.ts @@ -13,14 +13,14 @@ import type { PrivacyNoticeFramework } from "./PrivacyNoticeFramework"; */ export type PrivacyNoticeCreation = { name: string; - notice_key?: string; - internal_description?: string; + notice_key?: string | null; + internal_description?: string | null; consent_mechanism: ConsentMechanism; - data_uses?: Array; + data_uses?: Array | null; enforcement_level: EnforcementLevel; - disabled?: boolean; - has_gpc_flag?: boolean; - framework?: PrivacyNoticeFramework; - gpp_field_mapping?: Array; - translations?: Array; + disabled?: boolean | null; + has_gpc_flag?: boolean | null; + framework?: PrivacyNoticeFramework | null; + gpp_field_mapping?: Array | null; + translations?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyNoticeFramework.ts b/clients/admin-ui/src/types/api/models/PrivacyNoticeFramework.ts index 260756749e..9bbc33a671 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyNoticeFramework.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyNoticeFramework.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum PrivacyNoticeFramework { GPP_US_NATIONAL = "gpp_us_national", GPP_US_STATE = "gpp_us_state", diff --git a/clients/admin-ui/src/types/api/models/PrivacyNoticeRegion.ts b/clients/admin-ui/src/types/api/models/PrivacyNoticeRegion.ts index b220606a3f..1340b1a2f7 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyNoticeRegion.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyNoticeRegion.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum PrivacyNoticeRegion { ER = "er", DJ = "dj", diff --git a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts index b708418194..249314b9ab 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts @@ -19,20 +19,20 @@ import type { UserConsentPreference } from "./UserConsentPreference"; export type PrivacyNoticeResponse = { name: string; notice_key: string; - internal_description?: string; + internal_description?: string | null; consent_mechanism: ConsentMechanism; data_uses: Array; enforcement_level: EnforcementLevel; disabled: boolean; has_gpc_flag: boolean; - framework?: PrivacyNoticeFramework; - default_preference?: UserConsentPreference; + framework?: PrivacyNoticeFramework | null; + default_preference?: UserConsentPreference | null; id: string; - origin?: string; + origin?: string | null; created_at: string; updated_at: string; cookies: Array; systems_applicable?: boolean; translations?: Array; - gpp_field_mapping?: Array; + gpp_field_mapping?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithRegions.ts b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithRegions.ts index 3802eec8ed..5b44235df5 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithRegions.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithRegions.ts @@ -18,22 +18,22 @@ import type { UserConsentPreference } from "./UserConsentPreference"; export type PrivacyNoticeResponseWithRegions = { name: string; notice_key: string; - internal_description?: string; + internal_description?: string | null; consent_mechanism: ConsentMechanism; data_uses: Array; enforcement_level: EnforcementLevel; disabled: boolean; has_gpc_flag: boolean; - framework?: PrivacyNoticeFramework; - default_preference?: UserConsentPreference; + framework?: PrivacyNoticeFramework | null; + default_preference?: UserConsentPreference | null; id: string; - origin?: string; + origin?: string | null; created_at: string; updated_at: string; cookies: Array; systems_applicable?: boolean; translations?: Array; - gpp_field_mapping?: Array; + gpp_field_mapping?: Array | null; /** * A property calculated by observing which Experiences have linked this Notice */ diff --git a/clients/admin-ui/src/types/api/models/PrivacyNoticeUpdate.ts b/clients/admin-ui/src/types/api/models/PrivacyNoticeUpdate.ts index 1713a7dda4..23912281ea 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyNoticeUpdate.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyNoticeUpdate.ts @@ -15,15 +15,15 @@ import type { PrivacyNoticeFramework } from "./PrivacyNoticeFramework"; * Other aspects will be validated on a dry update after patch updates are applied against PrivacyNoticeCreation */ export type PrivacyNoticeUpdate = { - name?: string; - notice_key?: string; - internal_description?: string; - consent_mechanism?: ConsentMechanism; - data_uses?: Array; - enforcement_level?: EnforcementLevel; - disabled?: boolean; - has_gpc_flag?: boolean; - framework?: PrivacyNoticeFramework; - gpp_field_mapping?: Array; + name?: string | null; + notice_key?: string | null; + internal_description?: string | null; + consent_mechanism?: ConsentMechanism | null; + data_uses?: Array | null; + enforcement_level?: EnforcementLevel | null; + disabled?: boolean | null; + has_gpc_flag?: boolean | null; + framework?: PrivacyNoticeFramework | null; + gpp_field_mapping?: Array | null; translations: Array; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyPreferencesRequest.ts b/clients/admin-ui/src/types/api/models/PrivacyPreferencesRequest.ts index 774d17dbbb..6c04957cc2 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyPreferencesRequest.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyPreferencesRequest.ts @@ -30,16 +30,16 @@ export type PrivacyPreferencesRequest = { system_legitimate_interests_preferences?: Array; preferences?: Array; browser_identity: Identity; - code?: string; + code?: string | null; /** * If supplied, TC strings and AC strings are decoded and preferences saved for purpose_consent, purpose_legitimate_interests, vendor_consent, vendor_legitimate_interests, and special_features */ - fides_string?: string; - policy_key?: string; - privacy_experience_id?: string; - privacy_experience_config_history_id?: string; - user_geography?: string; - method?: ConsentMethod; - served_notice_history_id?: string; - property_id?: string; + fides_string?: string | null; + policy_key?: string | null; + privacy_experience_id?: string | null; + privacy_experience_config_history_id?: string | null; + user_geography?: string | null; + method?: ConsentMethod | null; + served_notice_history_id?: string | null; + property_id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestCreate.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestCreate.ts index b668ec4242..65ff3e1794 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestCreate.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestCreate.ts @@ -10,18 +10,18 @@ import type { Identity } from "./Identity"; * Data required to create a PrivacyRequest */ export type PrivacyRequestCreate = { - external_id?: string; - started_processing_at?: string; - finished_processing_at?: string; - requested_at?: string; + external_id?: string | null; + started_processing_at?: string | null; + finished_processing_at?: string | null; + requested_at?: string | null; identity: Identity; - consent_request_id?: string; + consent_request_id?: string | null; custom_privacy_request_fields?: Record< string, fides__api__schemas__redis_cache__CustomPrivacyRequestField - >; + > | null; policy_key: string; - encryption_key?: string; - property_id?: string; - consent_preferences?: Array; + encryption_key?: string | null; + property_id?: string | null; + consent_preferences?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestDRPStatusResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestDRPStatusResponse.ts index aaf28b4204..b6a74a4ad6 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestDRPStatusResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestDRPStatusResponse.ts @@ -10,9 +10,9 @@ import type { PrivacyRequestDRPStatus } from "./PrivacyRequestDRPStatus"; export type PrivacyRequestDRPStatusResponse = { request_id: string; received_at: string; - expected_by?: string; - processing_details?: string; + expected_by?: string | null; + processing_details?: string | null; status: PrivacyRequestDRPStatus; - reason?: string; - user_verification_url?: string; + reason?: string | null; + user_verification_url?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestFilter.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestFilter.ts index e8d8fd9b39..2f4a8cdab9 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestFilter.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestFilter.ts @@ -6,28 +6,25 @@ import type { ActionType } from "./ActionType"; import type { ColumnSort } from "./ColumnSort"; import type { PrivacyRequestStatus } from "./PrivacyRequestStatus"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type PrivacyRequestFilter = { - request_id?: string; - identities?: any; - custom_privacy_request_fields?: any; - status?: PrivacyRequestStatus | Array; - created_lt?: string; - created_gt?: string; - started_lt?: string; - started_gt?: string; - completed_lt?: string; - completed_gt?: string; - errored_lt?: string; - errored_gt?: string; - external_id?: string; - action_type?: ActionType; - verbose?: boolean; - include_identities?: boolean; - include_custom_privacy_request_fields?: boolean; - download_csv?: boolean; + request_id?: string | null; + identities?: null; + custom_privacy_request_fields?: null; + status?: PrivacyRequestStatus | Array | null; + created_lt?: string | null; + created_gt?: string | null; + started_lt?: string | null; + started_gt?: string | null; + completed_lt?: string | null; + completed_gt?: string | null; + errored_lt?: string | null; + errored_gt?: string | null; + external_id?: string | null; + action_type?: ActionType | null; + verbose?: boolean | null; + include_identities?: boolean | null; + include_custom_privacy_request_fields?: boolean | null; + download_csv?: boolean | null; sort_field?: string; sort_direction?: ColumnSort; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestNotificationInfo.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestNotificationInfo.ts index fedecb8f77..d18dc1eeef 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestNotificationInfo.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestNotificationInfo.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type PrivacyRequestNotificationInfo = { email_addresses: Array; notify_after_failures: number; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestOption.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestOption.ts index d7d6f0a749..1238f096e6 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestOption.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestOption.ts @@ -5,20 +5,17 @@ import type { fides__api__schemas__privacy_center_config__CustomPrivacyRequestField } from "./fides__api__schemas__privacy_center_config__CustomPrivacyRequestField"; import type { IdentityInputs } from "./IdentityInputs"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type PrivacyRequestOption = { policy_key: string; icon_path: string; title: string; description: string; - description_subtext?: Array; - confirmButtonText?: string; - cancelButtonText?: string; - identity_inputs?: IdentityInputs; + description_subtext?: Array | null; + confirmButtonText?: string | null; + cancelButtonText?: string | null; + identity_inputs?: IdentityInputs | null; custom_privacy_request_fields?: Record< string, fides__api__schemas__privacy_center_config__CustomPrivacyRequestField - >; + > | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestResponse.ts index 180c2200f2..f7da59430d 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestResponse.ts @@ -12,22 +12,22 @@ import type { PrivacyRequestStatus } from "./PrivacyRequestStatus"; */ export type PrivacyRequestResponse = { id: string; - created_at?: string; - started_processing_at?: string; - reviewed_at?: string; - reviewed_by?: string; - reviewer?: PrivacyRequestReviewer; - finished_processing_at?: string; - identity_verified_at?: string; - paused_at?: string; + created_at?: string | null; + started_processing_at?: string | null; + reviewed_at?: string | null; + reviewed_by?: string | null; + reviewer?: PrivacyRequestReviewer | null; + finished_processing_at?: string | null; + identity_verified_at?: string | null; + paused_at?: string | null; status: PrivacyRequestStatus; - external_id?: string; - identity?: Record; - custom_privacy_request_fields?: any; + external_id?: string | null; + identity?: Record | null; + custom_privacy_request_fields?: null; policy: PolicyResponse; - action_required_details?: CheckpointActionRequiredDetails; - resume_endpoint?: string; - days_left?: number; - custom_privacy_request_fields_approved_by?: string; - custom_privacy_request_fields_approved_at?: string; + action_required_details?: CheckpointActionRequiredDetails | null; + resume_endpoint?: string | null; + days_left?: number | null; + custom_privacy_request_fields_approved_by?: string | null; + custom_privacy_request_fields_approved_at?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestResumeFormat.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestResumeFormat.ts index c68134587c..a7d6dc806f 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestResumeFormat.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestResumeFormat.ts @@ -8,5 +8,5 @@ import type { Identity } from "./Identity"; * Expected request body to resume a privacy request after it was paused by a webhook */ export type PrivacyRequestResumeFormat = { - derived_identity?: Identity; + derived_identity?: Identity | null; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestTaskSchema.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestTaskSchema.ts index b961c03b11..b1190535a3 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestTaskSchema.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestTaskSchema.ts @@ -9,9 +9,9 @@ import type { ExecutionLogStatus } from "./ExecutionLogStatus"; * Schema for Privacy Request Tasks, which are individual nodes that are queued */ export type PrivacyRequestTaskSchema = { + status: ExecutionLogStatus; id: string; collection_address: string; - status: ExecutionLogStatus; created_at: string; updated_at: string; upstream_tasks: Array; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestVerboseResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestVerboseResponse.ts index 913e92311c..7c0e59dd10 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestVerboseResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestVerboseResponse.ts @@ -14,23 +14,23 @@ import type { PrivacyRequestStatus } from "./PrivacyRequestStatus"; */ export type PrivacyRequestVerboseResponse = { id: string; - created_at?: string; - started_processing_at?: string; - reviewed_at?: string; - reviewed_by?: string; - reviewer?: PrivacyRequestReviewer; - finished_processing_at?: string; - identity_verified_at?: string; - paused_at?: string; + created_at?: string | null; + started_processing_at?: string | null; + reviewed_at?: string | null; + reviewed_by?: string | null; + reviewer?: PrivacyRequestReviewer | null; + finished_processing_at?: string | null; + identity_verified_at?: string | null; + paused_at?: string | null; status: PrivacyRequestStatus; - external_id?: string; - identity?: Record; - custom_privacy_request_fields?: any; + external_id?: string | null; + identity?: Record | null; + custom_privacy_request_fields?: null; policy: PolicyResponse; - action_required_details?: CheckpointActionRequiredDetails; - resume_endpoint?: string; - days_left?: number; - custom_privacy_request_fields_approved_by?: string; - custom_privacy_request_fields_approved_at?: string; + action_required_details?: CheckpointActionRequiredDetails | null; + resume_endpoint?: string | null; + days_left?: number | null; + custom_privacy_request_fields_approved_by?: string | null; + custom_privacy_request_fields_approved_at?: string | null; results: Record>; }; diff --git a/clients/admin-ui/src/types/api/models/Property.ts b/clients/admin-ui/src/types/api/models/Property.ts index 402f8d8586..314055c827 100644 --- a/clients/admin-ui/src/types/api/models/Property.ts +++ b/clients/admin-ui/src/types/api/models/Property.ts @@ -20,10 +20,10 @@ import type { PropertyType } from "./PropertyType"; export type Property = { name: string; type: PropertyType; - id?: string; + id?: string | null; experiences: Array; - messaging_templates?: Array; - privacy_center_config?: PrivacyCenterConfig; - stylesheet?: string; + messaging_templates?: Array | null; + privacy_center_config?: PrivacyCenterConfig | null; + stylesheet?: string | null; paths: Array; }; diff --git a/clients/admin-ui/src/types/api/models/PropertyType.ts b/clients/admin-ui/src/types/api/models/PropertyType.ts index f0c1877421..2bc3f3de00 100644 --- a/clients/admin-ui/src/types/api/models/PropertyType.ts +++ b/clients/admin-ui/src/types/api/models/PropertyType.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum PropertyType { WEBSITE = "Website", OTHER = "Other", diff --git a/clients/admin-ui/src/types/api/models/ProviderEnum.ts b/clients/admin-ui/src/types/api/models/ProviderEnum.ts index 4cc9de5194..c504894db5 100644 --- a/clients/admin-ui/src/types/api/models/ProviderEnum.ts +++ b/clients/admin-ui/src/types/api/models/ProviderEnum.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum ProviderEnum { GOOGLE = "google", OKTA = "okta", diff --git a/clients/admin-ui/src/types/api/models/PublicPropertyResponse.ts b/clients/admin-ui/src/types/api/models/PublicPropertyResponse.ts index 6afef20449..ea250ac07e 100644 --- a/clients/admin-ui/src/types/api/models/PublicPropertyResponse.ts +++ b/clients/admin-ui/src/types/api/models/PublicPropertyResponse.ts @@ -22,10 +22,10 @@ import type { PropertyType } from "./PropertyType"; export type PublicPropertyResponse = { name: string; type: PropertyType; - id?: string; + id?: string | null; experiences: Array; - messaging_templates?: Array; - privacy_center_config?: PrivacyCenterConfig; - stylesheet?: string; + messaging_templates?: Array | null; + privacy_center_config?: PrivacyCenterConfig | null; + stylesheet?: string | null; paths: Array; }; diff --git a/clients/admin-ui/src/types/api/models/RateLimit.ts b/clients/admin-ui/src/types/api/models/RateLimit.ts index 37367ac6dc..a41fdbbe1b 100644 --- a/clients/admin-ui/src/types/api/models/RateLimit.ts +++ b/clients/admin-ui/src/types/api/models/RateLimit.ts @@ -10,5 +10,5 @@ import type { RateLimitPeriod } from "./RateLimitPeriod"; export type RateLimit = { rate: number; period: RateLimitPeriod; - custom_key?: string; + custom_key?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/RateLimitConfig.ts b/clients/admin-ui/src/types/api/models/RateLimitConfig.ts index 6a4e59b030..e7a9b3fe36 100644 --- a/clients/admin-ui/src/types/api/models/RateLimitConfig.ts +++ b/clients/admin-ui/src/types/api/models/RateLimitConfig.ts @@ -8,6 +8,6 @@ import type { RateLimit } from "./RateLimit"; * A config object which allows configuring rate limits for connectors */ export type RateLimitConfig = { - limits?: Array; - enabled?: boolean; + limits?: Array | null; + enabled?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/ReadSaaSRequest.ts b/clients/admin-ui/src/types/api/models/ReadSaaSRequest.ts index d74f5b3f7b..fa691de28a 100644 --- a/clients/admin-ui/src/types/api/models/ReadSaaSRequest.ts +++ b/clients/admin-ui/src/types/api/models/ReadSaaSRequest.ts @@ -16,21 +16,21 @@ import type { Strategy } from "./Strategy"; * that is used to format each collection result. */ export type ReadSaaSRequest = { - request_override?: string; - path?: string; - method?: HTTPMethod; - headers?: Array
; - query_params?: Array; - body?: string; - param_values?: Array; - client_config?: ClientConfig; - data_path?: string; - postprocessors?: Array; - pagination?: Strategy; - grouped_inputs?: Array; - ignore_errors?: boolean | Array; - rate_limit_config?: RateLimitConfig; - async_config?: AsyncConfig; - skip_missing_param_values?: boolean; - output?: string; + request_override?: string | null; + path?: string | null; + method?: HTTPMethod | null; + headers?: Array
| null; + query_params?: Array | null; + body?: string | null; + param_values?: Array | null; + client_config?: ClientConfig | null; + data_path?: string | null; + postprocessors?: Array | null; + pagination?: Strategy | null; + grouped_inputs?: Array | null; + ignore_errors?: boolean | Array | null; + rate_limit_config?: RateLimitConfig | null; + async_config?: AsyncConfig | null; + skip_missing_param_values?: boolean | null; + output?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/RecordConsentServedRequest.ts b/clients/admin-ui/src/types/api/models/RecordConsentServedRequest.ts index 23caeffe8c..3f040a6d79 100644 --- a/clients/admin-ui/src/types/api/models/RecordConsentServedRequest.ts +++ b/clients/admin-ui/src/types/api/models/RecordConsentServedRequest.ts @@ -20,12 +20,12 @@ export type RecordConsentServedRequest = { tcf_system_legitimate_interests?: Array; privacy_notice_history_ids?: Array; browser_identity: Identity; - code?: string; - privacy_experience_id?: string; - privacy_experience_config_history_id?: string; - user_geography?: string; - acknowledge_mode?: boolean; - served_notice_history_id?: string; + code?: string | null; + privacy_experience_id?: string | null; + privacy_experience_config_history_id?: string | null; + user_geography?: string | null; + acknowledge_mode?: boolean | null; + served_notice_history_id?: string | null; serving_component: ServingComponent; - property_id?: string; + property_id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts b/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts index e1b3f07e09..7ba577770e 100644 --- a/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts @@ -29,7 +29,7 @@ export type RedshiftDocsSchema = { /** * The default schema to be used for the database connection (defaults to public). */ - db_schema?: string; + db_schema?: string | null; /** * Indicates whether an SSH tunnel is required for the connection. Enable this option if your Redshift database is behind a firewall and requires SSH tunneling for remote connections. */ diff --git a/clients/admin-ui/src/types/api/models/Registration.ts b/clients/admin-ui/src/types/api/models/Registration.ts index ed2d4c83a7..258fdd0c52 100644 --- a/clients/admin-ui/src/types/api/models/Registration.ts +++ b/clients/admin-ui/src/types/api/models/Registration.ts @@ -8,6 +8,6 @@ export type Registration = { opt_in: boolean; analytics_id: string; - user_email?: string; - user_organization?: string; + user_email: string | null; + user_organization: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/RequestTaskCallbackRequest.ts b/clients/admin-ui/src/types/api/models/RequestTaskCallbackRequest.ts index 80d4eca810..d78b86913c 100644 --- a/clients/admin-ui/src/types/api/models/RequestTaskCallbackRequest.ts +++ b/clients/admin-ui/src/types/api/models/RequestTaskCallbackRequest.ts @@ -9,9 +9,9 @@ export type RequestTaskCallbackRequest = { /** * Access results collected asynchronously, as a list of rows. Use caution; this data may be used by dependent tasks downstream and/or uploaded to the end user. */ - access_results?: Array; + access_results?: null; /** * Number of records masked, as an integer */ - rows_masked?: number; + rows_masked?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/RuleCreate.ts b/clients/admin-ui/src/types/api/models/RuleCreate.ts index 7918dc5f6c..089d0f0259 100644 --- a/clients/admin-ui/src/types/api/models/RuleCreate.ts +++ b/clients/admin-ui/src/types/api/models/RuleCreate.ts @@ -11,8 +11,8 @@ import type { PolicyMaskingSpec } from "./PolicyMaskingSpec"; */ export type RuleCreate = { name: string; - key?: string; + key?: string | null; action_type: ActionType; - storage_destination_key?: string; - masking_strategy?: PolicyMaskingSpec; + storage_destination_key?: string | null; + masking_strategy?: PolicyMaskingSpec | null; }; diff --git a/clients/admin-ui/src/types/api/models/RuleResponse.ts b/clients/admin-ui/src/types/api/models/RuleResponse.ts index 93fb4e7b54..5dce7583c5 100644 --- a/clients/admin-ui/src/types/api/models/RuleResponse.ts +++ b/clients/admin-ui/src/types/api/models/RuleResponse.ts @@ -12,8 +12,8 @@ import type { StorageDestinationResponse } from "./StorageDestinationResponse"; */ export type RuleResponse = { name: string; - key?: string; + key?: string | null; action_type: ActionType; - storage_destination?: StorageDestinationResponse; - masking_strategy?: PolicyMaskingSpecResponse; + storage_destination?: StorageDestinationResponse | null; + masking_strategy?: PolicyMaskingSpecResponse | null; }; diff --git a/clients/admin-ui/src/types/api/models/RuleResponseWithTargets.ts b/clients/admin-ui/src/types/api/models/RuleResponseWithTargets.ts index e8d5557608..bce2967311 100644 --- a/clients/admin-ui/src/types/api/models/RuleResponseWithTargets.ts +++ b/clients/admin-ui/src/types/api/models/RuleResponseWithTargets.ts @@ -14,9 +14,9 @@ import type { StorageDestinationResponse } from "./StorageDestinationResponse"; */ export type RuleResponseWithTargets = { name: string; - key?: string; + key?: string | null; action_type: ActionType; - targets?: Array; - storage_destination?: StorageDestinationResponse; - masking_strategy?: PolicyMaskingSpecResponse; + targets?: Array | null; + storage_destination?: StorageDestinationResponse | null; + masking_strategy?: PolicyMaskingSpecResponse | null; }; diff --git a/clients/admin-ui/src/types/api/models/RuleTarget.ts b/clients/admin-ui/src/types/api/models/RuleTarget.ts index f847840bbb..b2b2976dd5 100644 --- a/clients/admin-ui/src/types/api/models/RuleTarget.ts +++ b/clients/admin-ui/src/types/api/models/RuleTarget.ts @@ -6,7 +6,7 @@ * An external representation of a Rule's target DataCategory within a Fidesops Policy */ export type RuleTarget = { - name?: string; - key?: string; + name?: string | null; + key?: string | null; data_category: string; }; diff --git a/clients/admin-ui/src/types/api/models/S3DocsSchema.ts b/clients/admin-ui/src/types/api/models/S3DocsSchema.ts index 6406964bf0..31ba3d0b94 100644 --- a/clients/admin-ui/src/types/api/models/S3DocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/S3DocsSchema.ts @@ -15,13 +15,13 @@ export type S3DocsSchema = { /** * Part of the credentials that provide access to your AWS account. This is required if using secret key authentication. */ - aws_access_key_id?: string; + aws_access_key_id?: string | null; /** * Part of the credentials that provide access to your AWS account. This is required if using secret key authentication. */ - aws_secret_access_key?: string; + aws_secret_access_key?: string | null; /** * If provided, the ARN of the role that should be assumed to connect to s3. */ - aws_assume_role_arn?: string; + aws_assume_role_arn?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/S3MonitorParams.ts b/clients/admin-ui/src/types/api/models/S3MonitorParams.ts index 5f9e0558af..a499d7262c 100644 --- a/clients/admin-ui/src/types/api/models/S3MonitorParams.ts +++ b/clients/admin-ui/src/types/api/models/S3MonitorParams.ts @@ -6,5 +6,5 @@ export type S3MonitorParams = { /** * The maximum number of objects that will be detected, per bucket */ - max_objects?: number; + max_objects?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/SaaSConfig.ts b/clients/admin-ui/src/types/api/models/SaaSConfig.ts index ed5cf64582..3a1aee6fbd 100644 --- a/clients/admin-ui/src/types/api/models/SaaSConfig.ts +++ b/clients/admin-ui/src/types/api/models/SaaSConfig.ts @@ -27,12 +27,12 @@ export type SaaSConfig = { version: string; replaceable?: boolean; connector_params: Array; - external_references?: Array; + external_references?: Array | null; client_config: ClientConfig; endpoints: Array; test_request: SaaSRequest; - data_protection_request?: SaaSRequest; - rate_limit_config?: RateLimitConfig; - consent_requests?: ConsentRequestMap; - user_guide?: string; + data_protection_request?: SaaSRequest | null; + rate_limit_config?: RateLimitConfig | null; + consent_requests?: ConsentRequestMap | null; + user_guide?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/SaaSConfigValidationDetails.ts b/clients/admin-ui/src/types/api/models/SaaSConfigValidationDetails.ts index 372ae7fbdc..9447f55f10 100644 --- a/clients/admin-ui/src/types/api/models/SaaSConfigValidationDetails.ts +++ b/clients/admin-ui/src/types/api/models/SaaSConfigValidationDetails.ts @@ -6,5 +6,5 @@ * Message with any validation issues with the SaaS config */ export type SaaSConfigValidationDetails = { - msg?: string; + msg?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/SaaSRequest.ts b/clients/admin-ui/src/types/api/models/SaaSRequest.ts index 39ff414063..9c75458ca4 100644 --- a/clients/admin-ui/src/types/api/models/SaaSRequest.ts +++ b/clients/admin-ui/src/types/api/models/SaaSRequest.ts @@ -18,20 +18,20 @@ import type { Strategy } from "./Strategy"; * Includes optional strategies for postprocessing and pagination. */ export type SaaSRequest = { - request_override?: string; - path?: string; - method?: HTTPMethod; - headers?: Array
; - query_params?: Array; - body?: string; - param_values?: Array; - client_config?: ClientConfig; - data_path?: string; - postprocessors?: Array; - pagination?: Strategy; - grouped_inputs?: Array; - ignore_errors?: boolean | Array; - rate_limit_config?: RateLimitConfig; - async_config?: AsyncConfig; - skip_missing_param_values?: boolean; + request_override?: string | null; + path?: string | null; + method?: HTTPMethod | null; + headers?: Array
| null; + query_params?: Array | null; + body?: string | null; + param_values?: Array | null; + client_config?: ClientConfig | null; + data_path?: string | null; + postprocessors?: Array | null; + pagination?: Strategy | null; + grouped_inputs?: Array | null; + ignore_errors?: boolean | Array | null; + rate_limit_config?: RateLimitConfig | null; + async_config?: AsyncConfig | null; + skip_missing_param_values?: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/SaaSRequestMap.ts b/clients/admin-ui/src/types/api/models/SaaSRequestMap.ts index fc1c5d4e29..bdd9deb42c 100644 --- a/clients/admin-ui/src/types/api/models/SaaSRequestMap.ts +++ b/clients/admin-ui/src/types/api/models/SaaSRequestMap.ts @@ -10,6 +10,6 @@ import type { SaaSRequest } from "./SaaSRequest"; */ export type SaaSRequestMap = { read?: ReadSaaSRequest | Array; - update?: SaaSRequest; - delete?: SaaSRequest; + update?: SaaSRequest | null; + delete?: SaaSRequest | null; }; diff --git a/clients/admin-ui/src/types/api/models/SaasConnectionTemplateValues.ts b/clients/admin-ui/src/types/api/models/SaasConnectionTemplateValues.ts index 06bcea50b4..bc9165f852 100644 --- a/clients/admin-ui/src/types/api/models/SaasConnectionTemplateValues.ts +++ b/clients/admin-ui/src/types/api/models/SaasConnectionTemplateValues.ts @@ -26,9 +26,9 @@ import type { TimescaleDocsSchema } from "./TimescaleDocsSchema"; * Schema with values to create both a Saas ConnectionConfig and DatasetConfig from a template */ export type SaasConnectionTemplateValues = { - name?: string; - key?: string; - description?: string; + name?: string | null; + key?: string | null; + description?: string | null; secrets: | MongoDBDocsSchema | PostgreSQLDocsSchema diff --git a/clients/admin-ui/src/types/api/models/SaasConnectionTemplateValuesExtended.ts b/clients/admin-ui/src/types/api/models/SaasConnectionTemplateValuesExtended.ts index 5e77081fbb..4e663a0180 100644 --- a/clients/admin-ui/src/types/api/models/SaasConnectionTemplateValuesExtended.ts +++ b/clients/admin-ui/src/types/api/models/SaasConnectionTemplateValuesExtended.ts @@ -23,13 +23,10 @@ import type { SnowflakeDocsSchema } from "./SnowflakeDocsSchema"; import type { SovrnDocsSchema } from "./SovrnDocsSchema"; import type { TimescaleDocsSchema } from "./TimescaleDocsSchema"; -/** - * Schema with values to create both a Saas ConnectionConfig and DatasetConfig from a template - */ export type SaasConnectionTemplateValuesExtended = { - name?: string; - key?: string; - description?: string; + name?: string | null; + key?: string | null; + description?: string | null; secrets: | MongoDBDocsSchema | PostgreSQLDocsSchema diff --git a/clients/admin-ui/src/types/api/models/Schema.ts b/clients/admin-ui/src/types/api/models/Schema.ts index d0f6dfe103..a2927b3fb5 100644 --- a/clients/admin-ui/src/types/api/models/Schema.ts +++ b/clients/admin-ui/src/types/api/models/Schema.ts @@ -5,26 +5,23 @@ import type { Classification } from "./Classification"; import type { DiffStatus } from "./DiffStatus"; -/** - * Base API model that represents a staged resource, fields common to all types of staged resources - */ export type Schema = { urn: string; user_assigned_data_categories?: Array; - name?: string; - description?: string; - monitor_config_id?: string; - updated_at?: string; - source_modified?: string; + name?: string | null; + description?: string | null; + monitor_config_id?: string | null; + updated_at?: string | null; + source_modified?: string | null; classifications?: Array; /** * The diff status of the staged resource */ - diff_status?: DiffStatus; + diff_status?: DiffStatus | null; /** * Represents the aggregate counts of diff statuses of the staged resource's children. This is computed 'on-demand', i.e. a specific instance method must be invoked to populate the field. */ child_diff_statuses?: Record; - database_name?: string; + database_name?: string | null; tables?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts b/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts index 5d97d1a289..ea10d836f8 100644 --- a/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts +++ b/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum ScopeRegistryEnum { ALLOW_LIST_CREATE = "allow_list:create", ALLOW_LIST_DELETE = "allow_list:delete", diff --git a/clients/admin-ui/src/types/api/models/ScyllaDocsSchema.ts b/clients/admin-ui/src/types/api/models/ScyllaDocsSchema.ts index 87d3e2b163..b8e9413ed6 100644 --- a/clients/admin-ui/src/types/api/models/ScyllaDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/ScyllaDocsSchema.ts @@ -23,7 +23,7 @@ export type ScyllaDocsSchema = { */ password: string; /** - * The keyspace used. + * The keyspace used. If not provided, DSRs for this integration will error. If the integration is used for D & D, then setting a keyspace is not required. */ - keyspace?: string; + keyspace?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/SecurityApplicationConfig.ts b/clients/admin-ui/src/types/api/models/SecurityApplicationConfig.ts index 009e6a60df..920583b324 100644 --- a/clients/admin-ui/src/types/api/models/SecurityApplicationConfig.ts +++ b/clients/admin-ui/src/types/api/models/SecurityApplicationConfig.ts @@ -2,12 +2,9 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type SecurityApplicationConfig = { /** * A list of client addresses allowed to communicate with the Fides webserver. */ - cors_origins?: Array; + cors_origins?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/SnowflakeDocsSchema.ts b/clients/admin-ui/src/types/api/models/SnowflakeDocsSchema.ts index 0036047ecd..b9fa1433c7 100644 --- a/clients/admin-ui/src/types/api/models/SnowflakeDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/SnowflakeDocsSchema.ts @@ -17,15 +17,15 @@ export type SnowflakeDocsSchema = { /** * The password used to authenticate and access the database. You can use a password or a private key, but not both. */ - password?: string; + password?: string | null; /** * The private key used to authenticate and access the database. If a `private_key_passphrase` is also provided, it is assumed to be encrypted; otherwise, it is assumed to be unencrypted. */ - private_key?: string; + private_key?: string | null; /** * The passphrase used for the encrypted private key. */ - private_key_passphrase?: string; + private_key_passphrase?: string | null; /** * The name of the Snowflake warehouse where your queries will be executed. */ @@ -41,5 +41,5 @@ export type SnowflakeDocsSchema = { /** * The Snowflake role to assume for the session, if different than Username. */ - role_name?: string; + role_name?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts b/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts index 3e7dfb60c7..92fda50812 100644 --- a/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts @@ -10,6 +10,6 @@ import type { AdvancedSettingsWithExtendedIdentityTypes } from "./AdvancedSettin export type SovrnDocsSchema = { third_party_vendor_name?: string; recipient_email_address?: string; - test_email_address?: string; + test_email_address?: string | null; advanced_settings?: AdvancedSettingsWithExtendedIdentityTypes; }; diff --git a/clients/admin-ui/src/types/api/models/StagedResource.ts b/clients/admin-ui/src/types/api/models/StagedResource.ts index c64c14a777..083ef9ab3f 100644 --- a/clients/admin-ui/src/types/api/models/StagedResource.ts +++ b/clients/admin-ui/src/types/api/models/StagedResource.ts @@ -11,16 +11,16 @@ import type { DiffStatus } from "./DiffStatus"; export type StagedResource = { urn: string; user_assigned_data_categories?: Array; - name?: string; - description?: string; - monitor_config_id?: string; - updated_at?: string; - source_modified?: string; + name?: string | null; + description?: string | null; + monitor_config_id?: string | null; + updated_at?: string | null; + source_modified?: string | null; classifications?: Array; /** * The diff status of the staged resource */ - diff_status?: DiffStatus; + diff_status?: DiffStatus | null; /** * Represents the aggregate counts of diff statuses of the staged resource's children. This is computed 'on-demand', i.e. a specific instance method must be invoked to populate the field. */ diff --git a/clients/admin-ui/src/types/api/models/StagedResourceAPIResponse.ts b/clients/admin-ui/src/types/api/models/StagedResourceAPIResponse.ts index 9d64263cfd..dea4064925 100644 --- a/clients/admin-ui/src/types/api/models/StagedResourceAPIResponse.ts +++ b/clients/admin-ui/src/types/api/models/StagedResourceAPIResponse.ts @@ -14,7 +14,7 @@ import type { DiffStatus } from "./DiffStatus"; * subclasses, since their structure is closely related to the ORM model and its fields, and provide a * more extensible way to represent staged resources in API responses (where we may want fields that * don't belong to the staged resources table, but are rather obtained from more complex queries, e.g - * joning against other tables). + * joining against other tables). * * This model adds the following "extra" fields that are not present on the stagedresources table: * - system: the name of the system related to the monitor, if applicable @@ -23,29 +23,29 @@ import type { DiffStatus } from "./DiffStatus"; export type StagedResourceAPIResponse = { urn: string; user_assigned_data_categories?: Array; - name?: string; - description?: string; - monitor_config_id?: string; - updated_at?: string; - source_modified?: string; + name?: string | null; + description?: string | null; + monitor_config_id?: string | null; + updated_at?: string | null; + source_modified?: string | null; classifications?: Array; /** * The diff status of the staged resource */ - diff_status?: DiffStatus; + diff_status?: DiffStatus | null; /** * Represents the aggregate counts of diff statuses of the staged resource's children. This is computed 'on-demand', i.e. a specific instance method must be invoked to populate the field. */ child_diff_statuses?: Record; - database_name?: string; - schema_name?: string; - parent_table_urn?: string; - table_name?: string; - data_type?: string; + database_name?: string | null; + schema_name?: string | null; + parent_table_urn?: string | null; + table_name?: string | null; + data_type?: string | null; fields?: Array; - num_rows?: number; + num_rows?: number | null; tables?: Array; schemas?: Array; - resource_type?: string; - system?: string; + resource_type?: string | null; + system?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/StorageApplicationConfig.ts b/clients/admin-ui/src/types/api/models/StorageApplicationConfig.ts index 1a837c8a71..6a3a44392c 100644 --- a/clients/admin-ui/src/types/api/models/StorageApplicationConfig.ts +++ b/clients/admin-ui/src/types/api/models/StorageApplicationConfig.ts @@ -4,9 +4,6 @@ import type { StorageTypeApiAccepted } from "./StorageTypeApiAccepted"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type StorageApplicationConfig = { active_default_storage_type: StorageTypeApiAccepted; }; diff --git a/clients/admin-ui/src/types/api/models/StorageConfigStatusMessage.ts b/clients/admin-ui/src/types/api/models/StorageConfigStatusMessage.ts index 1e6aeaff68..69c9ded8fd 100644 --- a/clients/admin-ui/src/types/api/models/StorageConfigStatusMessage.ts +++ b/clients/admin-ui/src/types/api/models/StorageConfigStatusMessage.ts @@ -8,6 +8,6 @@ import type { StorageConfigStatus } from "./StorageConfigStatus"; * A schema for checking configuration status of storage config. */ export type StorageConfigStatusMessage = { - config_status?: StorageConfigStatus; - detail?: string; + config_status?: StorageConfigStatus | null; + detail?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/StorageDestination.ts b/clients/admin-ui/src/types/api/models/StorageDestination.ts index c1149d88ef..666fb9a3d2 100644 --- a/clients/admin-ui/src/types/api/models/StorageDestination.ts +++ b/clients/admin-ui/src/types/api/models/StorageDestination.ts @@ -13,7 +13,7 @@ import type { StorageType } from "./StorageType"; export type StorageDestination = { type: StorageType; details: StorageDetailsS3 | StorageDetailsLocal; - format?: ResponseFormat; + format?: ResponseFormat | null; name: string; - key?: string; + key?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/StorageDestinationBase.ts b/clients/admin-ui/src/types/api/models/StorageDestinationBase.ts index a08746dc9f..ebae5d6734 100644 --- a/clients/admin-ui/src/types/api/models/StorageDestinationBase.ts +++ b/clients/admin-ui/src/types/api/models/StorageDestinationBase.ts @@ -13,5 +13,5 @@ import type { StorageType } from "./StorageType"; export type StorageDestinationBase = { type: StorageType; details: StorageDetailsS3 | StorageDetailsLocal; - format?: ResponseFormat; + format?: ResponseFormat | null; }; diff --git a/clients/admin-ui/src/types/api/models/StorageDetails.ts b/clients/admin-ui/src/types/api/models/StorageDetails.ts new file mode 100644 index 0000000000..53bf055a0f --- /dev/null +++ b/clients/admin-ui/src/types/api/models/StorageDetails.ts @@ -0,0 +1,13 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Enum for storage detail keys + */ +export enum StorageDetails { + BUCKET = "bucket", + NAMING = "naming", + MAX_RETRIES = "max_retries", + AUTH_METHOD = "auth_method", +} diff --git a/clients/admin-ui/src/types/api/models/StorageDetailsS3.ts b/clients/admin-ui/src/types/api/models/StorageDetailsS3.ts index b6e4a398d8..cd766b89b2 100644 --- a/clients/admin-ui/src/types/api/models/StorageDetailsS3.ts +++ b/clients/admin-ui/src/types/api/models/StorageDetailsS3.ts @@ -11,5 +11,5 @@ export type StorageDetailsS3 = { naming?: string; auth_method: AWSAuthMethod; bucket: string; - max_retries?: number; + max_retries?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/SupportedLanguage.ts b/clients/admin-ui/src/types/api/models/SupportedLanguage.ts index d9f326e78e..136975f94f 100644 --- a/clients/admin-ui/src/types/api/models/SupportedLanguage.ts +++ b/clients/admin-ui/src/types/api/models/SupportedLanguage.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum SupportedLanguage { AR = "ar", BG = "bg", diff --git a/clients/admin-ui/src/types/api/models/System.ts b/clients/admin-ui/src/types/api/models/System.ts index 5894619870..6dd2f728c9 100644 --- a/clients/admin-ui/src/types/api/models/System.ts +++ b/clients/admin-ui/src/types/api/models/System.ts @@ -23,19 +23,19 @@ export type System = { * Defines the Organization that this resource belongs to. */ organization_fides_key?: string; - tags?: Array; + tags?: Array | null; /** * Human-Readable name for this resource. */ - name?: string; + name?: string | null; /** * A detailed description of what this resource is. */ - description?: string; + description?: string | null; /** * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. */ - meta?: any; + meta?: null; /** * * The SystemMetadata resource model. @@ -43,7 +43,7 @@ export type System = { * Object used to hold application specific metadata for a system * */ - fidesctl_meta?: SystemMetadata; + fidesctl_meta?: SystemMetadata | null; /** * A required value to describe the type of system being modeled, examples include: Service, Application, Third Party, etc. */ @@ -51,11 +51,11 @@ export type System = { /** * The resources to which the system sends data. */ - egress?: Array; + egress?: Array | null; /** * The resources from which the system receives data. */ - ingress?: Array; + ingress?: Array | null; /** * * The PrivacyDeclaration resource model. @@ -68,19 +68,19 @@ export type System = { /** * An optional value to identify the owning department or group of the system within your organization */ - administrating_department?: string; + administrating_department?: string | null; /** * The unique identifier for the vendor that's associated with this system. */ - vendor_id?: string; + vendor_id?: string | null; /** * If specified, the unique identifier for the vendor that was previously associated with this system. */ - previous_vendor_id?: string; + previous_vendor_id?: string | null; /** * The deleted date of the vendor that's associated with this system. */ - vendor_deleted_date?: string; + vendor_deleted_date?: string | null; /** * Referenced Dataset fides keys used by the system. */ @@ -96,7 +96,7 @@ export type System = { /** * The reason that the system is exempt from privacy regulation. */ - reason_for_exemption?: string; + reason_for_exemption?: string | null; /** * Whether the vendor uses data to profile a consumer in a way that has a legal effect. */ @@ -120,23 +120,23 @@ export type System = { /** * Location where the DPAs or DIPAs can be found. */ - dpa_location?: string; + dpa_location?: string | null; /** * The optional status of a Data Protection Impact Assessment */ - dpa_progress?: string; + dpa_progress?: string | null; /** * A URL that points to the system's publicly accessible privacy policy. */ - privacy_policy?: string; + privacy_policy?: string | null; /** * The legal name for the business represented by the system. */ - legal_name?: string; + legal_name?: string | null; /** * The legal address for the business represented by the system. */ - legal_address?: string; + legal_address?: string | null; /** * * The model defining the responsibility or role over @@ -150,19 +150,19 @@ export type System = { /** * The official privacy contact address or DPO. */ - dpo?: string; + dpo?: string | null; /** * The party or parties that share the responsibility for processing personal data. */ - joint_controller_info?: string; + joint_controller_info?: string | null; /** * The data security practices employed by this system. */ - data_security_practices?: string; + data_security_practices?: string | null; /** * The maximum storage duration, in seconds, for cookies used by this system. */ - cookie_max_age_seconds?: number; + cookie_max_age_seconds?: number | null; /** * Whether this system uses cookie storage. */ @@ -178,9 +178,9 @@ export type System = { /** * A URL that points to the system's publicly accessible legitimate interest disclosure. */ - legitimate_interest_disclosure_url?: string; + legitimate_interest_disclosure_url?: string | null; /** * System-level cookies unassociated with a data use to deliver services and functionality */ - cookies?: Array; + cookies?: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/SystemHistoryResponse.ts b/clients/admin-ui/src/types/api/models/SystemHistoryResponse.ts index 9313b6a2b4..52896e90da 100644 --- a/clients/admin-ui/src/types/api/models/SystemHistoryResponse.ts +++ b/clients/admin-ui/src/types/api/models/SystemHistoryResponse.ts @@ -6,7 +6,7 @@ * Response schema for a single system history entry */ export type SystemHistoryResponse = { - edited_by?: string; + edited_by?: string | null; system_id: string; before: any; after: any; diff --git a/clients/admin-ui/src/types/api/models/SystemMetadata.ts b/clients/admin-ui/src/types/api/models/SystemMetadata.ts index 08451a936b..8d64d4e6f1 100644 --- a/clients/admin-ui/src/types/api/models/SystemMetadata.ts +++ b/clients/admin-ui/src/types/api/models/SystemMetadata.ts @@ -11,13 +11,13 @@ export type SystemMetadata = { /** * The external resource id for the system being modeled. */ - resource_id?: string; + resource_id?: string | null; /** * The host of the external resource for the system being modeled. */ - endpoint_address?: string; + endpoint_address?: string | null; /** * The port of the external resource for the system being modeled. */ - endpoint_port?: string; + endpoint_port?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/SystemResponse.ts b/clients/admin-ui/src/types/api/models/SystemResponse.ts index 9b2b89b872..d75d615e2f 100644 --- a/clients/admin-ui/src/types/api/models/SystemResponse.ts +++ b/clients/admin-ui/src/types/api/models/SystemResponse.ts @@ -27,19 +27,19 @@ export type SystemResponse = { * Defines the Organization that this resource belongs to. */ organization_fides_key?: string; - tags?: Array; + tags?: Array | null; /** * Human-Readable name for this resource. */ - name?: string; + name?: string | null; /** * A detailed description of what this resource is. */ - description?: string; + description?: string | null; /** * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. */ - meta?: any; + meta?: null; /** * * The SystemMetadata resource model. @@ -47,7 +47,7 @@ export type SystemResponse = { * Object used to hold application specific metadata for a system * */ - fidesctl_meta?: SystemMetadata; + fidesctl_meta?: SystemMetadata | null; /** * A required value to describe the type of system being modeled, examples include: Service, Application, Third Party, etc. */ @@ -55,11 +55,11 @@ export type SystemResponse = { /** * The resources to which the system sends data. */ - egress?: Array; + egress?: Array | null; /** * The resources from which the system receives data. */ - ingress?: Array; + ingress?: Array | null; /** * Extension of base pydantic model to include DB `id` field in the response */ @@ -67,19 +67,19 @@ export type SystemResponse = { /** * An optional value to identify the owning department or group of the system within your organization */ - administrating_department?: string; + administrating_department?: string | null; /** * The unique identifier for the vendor that's associated with this system. */ - vendor_id?: string; + vendor_id?: string | null; /** * If specified, the unique identifier for the vendor that was previously associated with this system. */ - previous_vendor_id?: string; + previous_vendor_id?: string | null; /** * The deleted date of the vendor that's associated with this system. */ - vendor_deleted_date?: string; + vendor_deleted_date?: string | null; /** * Referenced Dataset fides keys used by the system. */ @@ -95,7 +95,7 @@ export type SystemResponse = { /** * The reason that the system is exempt from privacy regulation. */ - reason_for_exemption?: string; + reason_for_exemption?: string | null; /** * Whether the vendor uses data to profile a consumer in a way that has a legal effect. */ @@ -119,23 +119,23 @@ export type SystemResponse = { /** * Location where the DPAs or DIPAs can be found. */ - dpa_location?: string; + dpa_location?: string | null; /** * The optional status of a Data Protection Impact Assessment */ - dpa_progress?: string; + dpa_progress?: string | null; /** * A URL that points to the system's publicly accessible privacy policy. */ - privacy_policy?: string; + privacy_policy?: string | null; /** * The legal name for the business represented by the system. */ - legal_name?: string; + legal_name?: string | null; /** * The legal address for the business represented by the system. */ - legal_address?: string; + legal_address?: string | null; /** * * The model defining the responsibility or role over @@ -149,19 +149,19 @@ export type SystemResponse = { /** * The official privacy contact address or DPO. */ - dpo?: string; + dpo?: string | null; /** * The party or parties that share the responsibility for processing personal data. */ - joint_controller_info?: string; + joint_controller_info?: string | null; /** * The data security practices employed by this system. */ - data_security_practices?: string; + data_security_practices?: string | null; /** * The maximum storage duration, in seconds, for cookies used by this system. */ - cookie_max_age_seconds?: number; + cookie_max_age_seconds?: number | null; /** * Whether this system uses cookie storage. */ @@ -177,20 +177,20 @@ export type SystemResponse = { /** * A URL that points to the system's publicly accessible legitimate interest disclosure. */ - legitimate_interest_disclosure_url?: string; + legitimate_interest_disclosure_url?: string | null; /** * System-level cookies unassociated with a data use to deliver services and functionality */ - cookies?: Array; + cookies?: Array | null; created_at: string; /** * * Describes the returned schema for a ConnectionConfiguration. * */ - connection_configs?: ConnectionConfigurationResponse; + connection_configs: ConnectionConfigurationResponse | null; /** * System managers of the current system */ - data_stewards?: Array; + data_stewards: Array | null; }; diff --git a/clients/admin-ui/src/types/api/models/SystemScannerStatus.ts b/clients/admin-ui/src/types/api/models/SystemScannerStatus.ts index 79a3b1705f..c816da2ab8 100644 --- a/clients/admin-ui/src/types/api/models/SystemScannerStatus.ts +++ b/clients/admin-ui/src/types/api/models/SystemScannerStatus.ts @@ -9,6 +9,6 @@ import type { ClusterHealth } from "./ClusterHealth"; */ export type SystemScannerStatus = { enabled?: boolean; - cluster_health?: ClusterHealth; - cluster_error?: string; + cluster_health?: ClusterHealth | null; + cluster_error?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/SystemSummary.ts b/clients/admin-ui/src/types/api/models/SystemSummary.ts index 4e65d7323f..270a81d02e 100644 --- a/clients/admin-ui/src/types/api/models/SystemSummary.ts +++ b/clients/admin-ui/src/types/api/models/SystemSummary.ts @@ -8,7 +8,7 @@ export type SystemSummary = { id: string; fides_key: string; - vendor_id?: string; + vendor_id?: string | null; name: string; purposes: number; cookies: number; diff --git a/clients/admin-ui/src/types/api/models/SystemType.ts b/clients/admin-ui/src/types/api/models/SystemType.ts index 0d4200f587..b08e9d168e 100644 --- a/clients/admin-ui/src/types/api/models/SystemType.ts +++ b/clients/admin-ui/src/types/api/models/SystemType.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum SystemType { SAAS = "saas", DATABASE = "database", diff --git a/clients/admin-ui/src/types/api/models/TCDecode.ts b/clients/admin-ui/src/types/api/models/TCDecode.ts index 2718bd9028..9a83dc1d9d 100644 --- a/clients/admin-ui/src/types/api/models/TCDecode.ts +++ b/clients/admin-ui/src/types/api/models/TCDecode.ts @@ -8,5 +8,5 @@ import type { TCMobileData } from "./TCMobileData"; * Decode response schema for returning TC Mobile Data */ export type TCDecode = { - fides_mobile_data?: TCMobileData; + fides_mobile_data?: TCMobileData | null; }; diff --git a/clients/admin-ui/src/types/api/models/TCFFeatureRecord.ts b/clients/admin-ui/src/types/api/models/TCFFeatureRecord.ts index 2958e34f84..27df382c81 100644 --- a/clients/admin-ui/src/types/api/models/TCFFeatureRecord.ts +++ b/clients/admin-ui/src/types/api/models/TCFFeatureRecord.ts @@ -21,7 +21,7 @@ export type TCFFeatureRecord = { * Description of the GVL feature or special feature. */ description: string; - default_preference?: UserConsentPreference; + default_preference?: UserConsentPreference | null; vendors?: Array; systems?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/TCFPurposeConsentRecord.ts b/clients/admin-ui/src/types/api/models/TCFPurposeConsentRecord.ts index a39f7b28e4..00f8c4b877 100644 --- a/clients/admin-ui/src/types/api/models/TCFPurposeConsentRecord.ts +++ b/clients/admin-ui/src/types/api/models/TCFPurposeConsentRecord.ts @@ -29,7 +29,7 @@ export type TCFPurposeConsentRecord = { * The fideslang default taxonomy data uses that are associated with the purpose. */ data_uses: Array; - default_preference?: UserConsentPreference; + default_preference?: UserConsentPreference | null; vendors?: Array; systems?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/TCFPurposeLegitimateInterestsRecord.ts b/clients/admin-ui/src/types/api/models/TCFPurposeLegitimateInterestsRecord.ts index a7cdb1e392..122b45500d 100644 --- a/clients/admin-ui/src/types/api/models/TCFPurposeLegitimateInterestsRecord.ts +++ b/clients/admin-ui/src/types/api/models/TCFPurposeLegitimateInterestsRecord.ts @@ -29,7 +29,7 @@ export type TCFPurposeLegitimateInterestsRecord = { * The fideslang default taxonomy data uses that are associated with the purpose. */ data_uses: Array; - default_preference?: UserConsentPreference; + default_preference?: UserConsentPreference | null; vendors?: Array; systems?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/TCFPurposeOverrideSchema.ts b/clients/admin-ui/src/types/api/models/TCFPurposeOverrideSchema.ts index 53f4cfd769..e91662cb07 100644 --- a/clients/admin-ui/src/types/api/models/TCFPurposeOverrideSchema.ts +++ b/clients/admin-ui/src/types/api/models/TCFPurposeOverrideSchema.ts @@ -9,6 +9,6 @@ import type { TCFLegalBasisEnum } from "./TCFLegalBasisEnum"; */ export type TCFPurposeOverrideSchema = { purpose: number; - is_included?: boolean; - required_legal_basis?: TCFLegalBasisEnum; + is_included?: boolean | null; + required_legal_basis?: TCFLegalBasisEnum | null; }; diff --git a/clients/admin-ui/src/types/api/models/TCFSpecialFeatureRecord.ts b/clients/admin-ui/src/types/api/models/TCFSpecialFeatureRecord.ts index ac853ee8ff..d823ff3079 100644 --- a/clients/admin-ui/src/types/api/models/TCFSpecialFeatureRecord.ts +++ b/clients/admin-ui/src/types/api/models/TCFSpecialFeatureRecord.ts @@ -21,7 +21,7 @@ export type TCFSpecialFeatureRecord = { * Description of the GVL feature or special feature. */ description: string; - default_preference?: UserConsentPreference; + default_preference?: UserConsentPreference | null; vendors?: Array; systems?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/TCFSpecialPurposeRecord.ts b/clients/admin-ui/src/types/api/models/TCFSpecialPurposeRecord.ts index 2f268e2128..ddfce6108e 100644 --- a/clients/admin-ui/src/types/api/models/TCFSpecialPurposeRecord.ts +++ b/clients/admin-ui/src/types/api/models/TCFSpecialPurposeRecord.ts @@ -29,7 +29,7 @@ export type TCFSpecialPurposeRecord = { * The fideslang default taxonomy data uses that are associated with the purpose. */ data_uses: Array; - default_preference?: UserConsentPreference; + default_preference?: UserConsentPreference | null; vendors?: Array; systems?: Array; legal_bases?: Array; diff --git a/clients/admin-ui/src/types/api/models/TCFVendorConsentRecord.ts b/clients/admin-ui/src/types/api/models/TCFVendorConsentRecord.ts index fc4d922b6b..8f22b0cc86 100644 --- a/clients/admin-ui/src/types/api/models/TCFVendorConsentRecord.ts +++ b/clients/admin-ui/src/types/api/models/TCFVendorConsentRecord.ts @@ -10,10 +10,10 @@ import type { UserConsentPreference } from "./UserConsentPreference"; */ export type TCFVendorConsentRecord = { id: string; - has_vendor_id?: boolean; - name?: string; - description?: string; - vendor_deleted_date?: string; - default_preference?: UserConsentPreference; + has_vendor_id?: boolean | null; + name?: string | null; + description?: string | null; + vendor_deleted_date?: string | null; + default_preference?: UserConsentPreference | null; purpose_consents?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/TCFVendorLegitimateInterestsRecord.ts b/clients/admin-ui/src/types/api/models/TCFVendorLegitimateInterestsRecord.ts index bcaa98dc97..c5dc56aea7 100644 --- a/clients/admin-ui/src/types/api/models/TCFVendorLegitimateInterestsRecord.ts +++ b/clients/admin-ui/src/types/api/models/TCFVendorLegitimateInterestsRecord.ts @@ -10,10 +10,10 @@ import type { UserConsentPreference } from "./UserConsentPreference"; */ export type TCFVendorLegitimateInterestsRecord = { id: string; - has_vendor_id?: boolean; - name?: string; - description?: string; - vendor_deleted_date?: string; - default_preference?: UserConsentPreference; + has_vendor_id?: boolean | null; + name?: string | null; + description?: string | null; + vendor_deleted_date?: string | null; + default_preference?: UserConsentPreference | null; purpose_legitimate_interests?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/TCFVendorRelationships.ts b/clients/admin-ui/src/types/api/models/TCFVendorRelationships.ts index 34e86d3c37..6734c7249b 100644 --- a/clients/admin-ui/src/types/api/models/TCFVendorRelationships.ts +++ b/clients/admin-ui/src/types/api/models/TCFVendorRelationships.ts @@ -10,17 +10,17 @@ import type { EmbeddedPurpose } from "./EmbeddedPurpose"; */ export type TCFVendorRelationships = { id: string; - has_vendor_id?: boolean; - name?: string; - description?: string; - vendor_deleted_date?: string; + has_vendor_id?: boolean | null; + name?: string | null; + description?: string | null; + vendor_deleted_date?: string | null; special_purposes?: Array; features?: Array; special_features?: Array; - cookie_max_age_seconds?: number; - uses_cookies?: boolean; - cookie_refresh?: boolean; - uses_non_cookie_access?: boolean; - legitimate_interest_disclosure_url?: string; - privacy_policy_url?: string; + cookie_max_age_seconds?: number | null; + uses_cookies?: boolean | null; + cookie_refresh?: boolean | null; + uses_non_cookie_access?: boolean | null; + legitimate_interest_disclosure_url?: string | null; + privacy_policy_url?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/TCMobileData.ts b/clients/admin-ui/src/types/api/models/TCMobileData.ts index a56199facb..b46ec7a7da 100644 --- a/clients/admin-ui/src/types/api/models/TCMobileData.ts +++ b/clients/admin-ui/src/types/api/models/TCMobileData.ts @@ -11,84 +11,58 @@ export type TCMobileData = { /** * The unsigned integer ID of CMP SDK */ - IABTCF_CmpSdkID?: number; + IABTCF_CmpSdkID?: number | null; /** * The unsigned integer version number of CMP SDK */ - IABTCF_CmpSdkVersion?: number; + IABTCF_CmpSdkVersion?: number | null; /** * The unsigned integer representing the version of the TCF that these consents adhere to. */ - IABTCF_PolicyVersion?: number; + IABTCF_PolicyVersion?: number | null; /** * 1: GDPR applies in current context, 0 - GDPR does not apply in current context, None=undetermined */ - IABTCF_gdprApplies?: TCMobileData.IABTCF_gdprApplies; + IABTCF_gdprApplies?: number | null; /** * Two-letter ISO 3166-1 alpha-2 code */ - IABTCF_PublisherCC?: string; + IABTCF_PublisherCC?: string | null; /** * Vendors can use this value to determine whether consent for purpose one is required. 0: no special treatment. 1: purpose one not disclosed */ - IABTCF_PurposeOneTreatment?: TCMobileData.IABTCF_PurposeOneTreatment; + IABTCF_PurposeOneTreatment?: number | null; /** * 1 - CMP uses customized stack descriptions and/or modified or supplemented standard illustrations.0 - CMP did not use a non-standard stack desc. and/or modified or supplemented Illustrations */ - IABTCF_UseNonStandardTexts?: TCMobileData.IABTCF_UseNonStandardTexts; + IABTCF_UseNonStandardTexts?: number | null; /** * Fully encoded TC string */ - IABTCF_TCString?: string; + IABTCF_TCString: string | null; /** * Binary string: The '0' or '1' at position n – where n's indexing begins at 0 – indicates the consent status for Vendor ID n+1; false and true respectively. eg. '1' at index 0 is consent true for vendor ID 1 */ - IABTCF_VendorConsents?: string; + IABTCF_VendorConsents: string | null; /** * Binary String: The '0' or '1' at position n – where n's indexing begins at 0 – indicates the legitimate interest status for Vendor ID n+1; false and true respectively. eg. '1' at index 0 is legitimate interest established true for vendor ID 1 */ - IABTCF_VendorLegitimateInterests?: string; + IABTCF_VendorLegitimateInterests?: string | null; /** * Binary String: The '0' or '1' at position n – where n's indexing begins at 0 – indicates the consent status for purpose ID n+1; false and true respectively. eg. '1' at index 0 is consent true for purpose ID 1 */ - IABTCF_PurposeConsents?: string; + IABTCF_PurposeConsents?: string | null; /** * Binary String: The '0' or '1' at position n – where n's indexing begins at 0 – indicates the legitimate interest status for purpose ID n+1; false and true respectively. eg. '1' at index 0 is legitimate interest established true for purpose ID 1 */ - IABTCF_PurposeLegitimateInterests?: string; + IABTCF_PurposeLegitimateInterests?: string | null; /** * Binary String: The '0' or '1' at position n – where n's indexing begins at 0 – indicates the opt-in status for special feature ID n+1; false and true respectively. eg. '1' at index 0 is opt-in true for special feature ID 1 */ - IABTCF_SpecialFeaturesOptIns?: string; - IABTCF_PublisherConsent?: string; - IABTCF_PublisherLegitimateInterests?: string; - IABTCF_PublisherCustomPurposesConsents?: string; - IABTCF_PublisherCustomPurposesLegitimateInterests?: string; - IABTCF_AddtlConsent?: string; + IABTCF_SpecialFeaturesOptIns?: string | null; + IABTCF_PublisherConsent?: string | null; + IABTCF_PublisherLegitimateInterests?: string | null; + IABTCF_PublisherCustomPurposesConsents?: string | null; + IABTCF_PublisherCustomPurposesLegitimateInterests?: string | null; + IABTCF_AddtlConsent?: string | null; }; - -export namespace TCMobileData { - /** - * 1: GDPR applies in current context, 0 - GDPR does not apply in current context, None=undetermined - */ - export enum IABTCF_gdprApplies { - "_0" = 0, - "_1" = 1, - } - - /** - * Vendors can use this value to determine whether consent for purpose one is required. 0: no special treatment. 1: purpose one not disclosed - */ - export enum IABTCF_PurposeOneTreatment { - "_0" = 0, - "_1" = 1, - } - - /** - * 1 - CMP uses customized stack descriptions and/or modified or supplemented standard illustrations.0 - CMP did not use a non-standard stack desc. and/or modified or supplemented Illustrations - */ - export enum IABTCF_UseNonStandardTexts { - "_0" = 0, - "_1" = 1, - } -} diff --git a/clients/admin-ui/src/types/api/models/Table.ts b/clients/admin-ui/src/types/api/models/Table.ts index 96b0e47087..605bcc18ba 100644 --- a/clients/admin-ui/src/types/api/models/Table.ts +++ b/clients/admin-ui/src/types/api/models/Table.ts @@ -5,29 +5,26 @@ import type { Classification } from "./Classification"; import type { DiffStatus } from "./DiffStatus"; -/** - * Base API model that represents a staged resource, fields common to all types of staged resources - */ export type Table = { urn: string; user_assigned_data_categories?: Array; - name?: string; - description?: string; - monitor_config_id?: string; - updated_at?: string; - source_modified?: string; + name?: string | null; + description?: string | null; + monitor_config_id?: string | null; + updated_at?: string | null; + source_modified?: string | null; classifications?: Array; /** * The diff status of the staged resource */ - diff_status?: DiffStatus; + diff_status?: DiffStatus | null; /** * Represents the aggregate counts of diff statuses of the staged resource's children. This is computed 'on-demand', i.e. a specific instance method must be invoked to populate the field. */ child_diff_statuses?: Record; - database_name?: string; + database_name?: string | null; schema_name: string; parent_schema: string; fields?: Array; - num_rows?: number; + num_rows?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/TestMessagingStatusMessage.ts b/clients/admin-ui/src/types/api/models/TestMessagingStatusMessage.ts index 9a3b86407f..85b91a835f 100644 --- a/clients/admin-ui/src/types/api/models/TestMessagingStatusMessage.ts +++ b/clients/admin-ui/src/types/api/models/TestMessagingStatusMessage.ts @@ -9,6 +9,6 @@ import type { MessagingConnectionTestStatus } from "./MessagingConnectionTestSta */ export type TestMessagingStatusMessage = { msg: string; - test_status?: MessagingConnectionTestStatus; - failure_reason?: string; + test_status?: MessagingConnectionTestStatus | null; + failure_reason?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/TestStatus.ts b/clients/admin-ui/src/types/api/models/TestStatus.ts index 8bc38a4c1a..30a7691c32 100644 --- a/clients/admin-ui/src/types/api/models/TestStatus.ts +++ b/clients/admin-ui/src/types/api/models/TestStatus.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum TestStatus { PASSED = "passed", FAILED = "failed", diff --git a/clients/admin-ui/src/types/api/models/TestStatusMessage.ts b/clients/admin-ui/src/types/api/models/TestStatusMessage.ts index 5e0d295475..1dcf244493 100644 --- a/clients/admin-ui/src/types/api/models/TestStatusMessage.ts +++ b/clients/admin-ui/src/types/api/models/TestStatusMessage.ts @@ -9,6 +9,6 @@ import type { ConnectionTestStatus } from "./ConnectionTestStatus"; */ export type TestStatusMessage = { msg: string; - test_status?: ConnectionTestStatus; - failure_reason?: string; + test_status?: ConnectionTestStatus | null; + failure_reason?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts b/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts index 48d0b1858d..52abf320fb 100644 --- a/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts @@ -17,11 +17,11 @@ export type TimescaleDocsSchema = { /** * The user account used to authenticate and access the database. */ - username?: string; + username?: string | null; /** * The password used to authenticate and access the database. */ - password?: string; + password?: string | null; /** * The name of the specific database within the database server that you want to connect to. */ @@ -29,7 +29,7 @@ export type TimescaleDocsSchema = { /** * The default schema to be used for the database connection (defaults to public). */ - db_schema?: string; + db_schema?: string | null; /** * Indicates whether an SSH tunnel is required for the connection. Enable this option if your PostgreSQL server is behind a firewall and requires SSH tunneling for remote connections. */ diff --git a/clients/admin-ui/src/types/api/models/UserConsentPreference.ts b/clients/admin-ui/src/types/api/models/UserConsentPreference.ts index 35cec38a7e..28ce17a716 100644 --- a/clients/admin-ui/src/types/api/models/UserConsentPreference.ts +++ b/clients/admin-ui/src/types/api/models/UserConsentPreference.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * An enumeration. - */ export enum UserConsentPreference { OPT_IN = "opt_in", OPT_OUT = "opt_out", diff --git a/clients/admin-ui/src/types/api/models/UserCreate.ts b/clients/admin-ui/src/types/api/models/UserCreate.ts index aa299ff717..38328b863f 100644 --- a/clients/admin-ui/src/types/api/models/UserCreate.ts +++ b/clients/admin-ui/src/types/api/models/UserCreate.ts @@ -7,9 +7,9 @@ */ export type UserCreate = { username: string; - password?: string; + password?: string | null; email_address: string; - first_name?: string; - last_name?: string; + first_name?: string | null; + last_name?: string | null; disabled?: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/UserPermissionsEdit.ts b/clients/admin-ui/src/types/api/models/UserPermissionsEdit.ts index 0b03104581..0d75ae3c98 100644 --- a/clients/admin-ui/src/types/api/models/UserPermissionsEdit.ts +++ b/clients/admin-ui/src/types/api/models/UserPermissionsEdit.ts @@ -9,5 +9,5 @@ import type { RoleRegistryEnum } from "./RoleRegistryEnum"; */ export type UserPermissionsEdit = { roles: Array; - id?: string; + id?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/UserResponse.ts b/clients/admin-ui/src/types/api/models/UserResponse.ts index c424be6d56..d4694acc88 100644 --- a/clients/admin-ui/src/types/api/models/UserResponse.ts +++ b/clients/admin-ui/src/types/api/models/UserResponse.ts @@ -9,9 +9,9 @@ export type UserResponse = { id: string; username: string; created_at: string; - email_address?: string; - first_name?: string; - last_name?: string; - disabled?: boolean; - disabled_reason?: string; + email_address: string | null; + first_name?: string | null; + last_name?: string | null; + disabled?: boolean | null; + disabled_reason?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/UserUpdate.ts b/clients/admin-ui/src/types/api/models/UserUpdate.ts index 4bded69a43..509ff5b423 100644 --- a/clients/admin-ui/src/types/api/models/UserUpdate.ts +++ b/clients/admin-ui/src/types/api/models/UserUpdate.ts @@ -6,7 +6,7 @@ * Data required to update a FidesUser */ export type UserUpdate = { - email_address?: string; - first_name?: string; - last_name?: string; + email_address?: string | null; + first_name?: string | null; + last_name?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/Violation.ts b/clients/admin-ui/src/types/api/models/Violation.ts index d9868caf37..0207513c18 100644 --- a/clients/admin-ui/src/types/api/models/Violation.ts +++ b/clients/admin-ui/src/types/api/models/Violation.ts @@ -8,9 +8,6 @@ import type { ViolationAttributes } from "./ViolationAttributes"; * The model for violations within an evaluation. */ export type Violation = { - /** - * The model for attributes which led to an evaluation violation - */ violating_attributes: ViolationAttributes; /** * A human-readable string detailing the evaluation violation. diff --git a/clients/admin-ui/src/types/api/models/fides__api__schemas__application_config__ConsentConfig.ts b/clients/admin-ui/src/types/api/models/fides__api__schemas__application_config__ConsentConfig.ts index b893d05840..7dc400a716 100644 --- a/clients/admin-ui/src/types/api/models/fides__api__schemas__application_config__ConsentConfig.ts +++ b/clients/admin-ui/src/types/api/models/fides__api__schemas__application_config__ConsentConfig.ts @@ -2,9 +2,6 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type fides__api__schemas__application_config__ConsentConfig = { - override_vendor_purposes?: boolean; + override_vendor_purposes: boolean | null; }; diff --git a/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_bigquery__KeyfileCreds.ts b/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_bigquery__KeyfileCreds.ts index bed2debeed..36b525f259 100644 --- a/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_bigquery__KeyfileCreds.ts +++ b/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_bigquery__KeyfileCreds.ts @@ -7,14 +7,14 @@ */ export type fides__api__schemas__connection_configuration__connection_secrets_bigquery__KeyfileCreds = { - type?: string; + type?: string | null; project_id: string; - private_key_id?: string; - private_key?: string; - client_email?: string; - client_id?: string; - auth_uri?: string; - token_uri?: string; - auth_provider_x509_cert_url?: string; - client_x509_cert_url?: string; + private_key_id?: string | null; + private_key?: string | null; + client_email?: string | null; + client_id?: string | null; + auth_uri?: string | null; + token_uri?: string | null; + auth_provider_x509_cert_url?: string | null; + client_x509_cert_url?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_google_cloud_sql_mysql__KeyfileCreds.ts b/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_google_cloud_sql_mysql__KeyfileCreds.ts index b194e9acc2..d4c4e70d2e 100644 --- a/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_google_cloud_sql_mysql__KeyfileCreds.ts +++ b/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_google_cloud_sql_mysql__KeyfileCreds.ts @@ -7,15 +7,15 @@ */ export type fides__api__schemas__connection_configuration__connection_secrets_google_cloud_sql_mysql__KeyfileCreds = { - type?: string; + type?: string | null; project_id: string; - private_key_id?: string; - private_key?: string; - client_email?: string; - client_id?: string; - auth_uri?: string; - token_uri?: string; - auth_provider_x509_cert_url?: string; - client_x509_cert_url?: string; + private_key_id?: string | null; + private_key?: string | null; + client_email?: string | null; + client_id?: string | null; + auth_uri?: string | null; + token_uri?: string | null; + auth_provider_x509_cert_url?: string | null; + client_x509_cert_url?: string | null; universe_domain: string; }; diff --git a/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_google_cloud_sql_postgres__KeyfileCreds.ts b/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_google_cloud_sql_postgres__KeyfileCreds.ts index 34a215b9ca..8affb57a12 100644 --- a/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_google_cloud_sql_postgres__KeyfileCreds.ts +++ b/clients/admin-ui/src/types/api/models/fides__api__schemas__connection_configuration__connection_secrets_google_cloud_sql_postgres__KeyfileCreds.ts @@ -7,15 +7,15 @@ */ export type fides__api__schemas__connection_configuration__connection_secrets_google_cloud_sql_postgres__KeyfileCreds = { - type?: string; + type?: string | null; project_id: string; - private_key_id?: string; - private_key?: string; - client_email?: string; - client_id?: string; - auth_uri?: string; - token_uri?: string; - auth_provider_x509_cert_url?: string; - client_x509_cert_url?: string; + private_key_id?: string | null; + private_key?: string | null; + client_email?: string | null; + client_id?: string | null; + auth_uri?: string | null; + token_uri?: string | null; + auth_provider_x509_cert_url?: string | null; + client_x509_cert_url?: string | null; universe_domain: string; }; diff --git a/clients/admin-ui/src/types/api/models/fides__api__schemas__policy__Policy.ts b/clients/admin-ui/src/types/api/models/fides__api__schemas__policy__Policy.ts index 06b9e85b5a..64cdfd4171 100644 --- a/clients/admin-ui/src/types/api/models/fides__api__schemas__policy__Policy.ts +++ b/clients/admin-ui/src/types/api/models/fides__api__schemas__policy__Policy.ts @@ -9,7 +9,7 @@ import type { DrpAction } from "./DrpAction"; */ export type fides__api__schemas__policy__Policy = { name: string; - key?: string; - drp_action?: DrpAction; - execution_timeframe?: number; + key?: string | null; + drp_action?: DrpAction | null; + execution_timeframe?: number | null; }; diff --git a/clients/admin-ui/src/types/api/models/fides__api__schemas__privacy_center_config__ConsentConfig.ts b/clients/admin-ui/src/types/api/models/fides__api__schemas__privacy_center_config__ConsentConfig.ts index cfe20f063a..704fa77da5 100644 --- a/clients/admin-ui/src/types/api/models/fides__api__schemas__privacy_center_config__ConsentConfig.ts +++ b/clients/admin-ui/src/types/api/models/fides__api__schemas__privacy_center_config__ConsentConfig.ts @@ -5,9 +5,6 @@ import type { ConsentConfigButton } from "./ConsentConfigButton"; import type { ConsentConfigPage } from "./ConsentConfigPage"; -/** - * A base template for all other Fides Schemas to inherit from. - */ export type fides__api__schemas__privacy_center_config__ConsentConfig = { button: ConsentConfigButton; page: ConsentConfigPage; diff --git a/clients/admin-ui/src/types/api/models/fides__api__schemas__privacy_center_config__CustomPrivacyRequestField.ts b/clients/admin-ui/src/types/api/models/fides__api__schemas__privacy_center_config__CustomPrivacyRequestField.ts index aa07be7392..ee7744fe3f 100644 --- a/clients/admin-ui/src/types/api/models/fides__api__schemas__privacy_center_config__CustomPrivacyRequestField.ts +++ b/clients/admin-ui/src/types/api/models/fides__api__schemas__privacy_center_config__CustomPrivacyRequestField.ts @@ -2,14 +2,11 @@ /* tslint:disable */ /* eslint-disable */ -/** - * A base template for all other Fides Schemas to inherit from. - */ export type fides__api__schemas__privacy_center_config__CustomPrivacyRequestField = { label: string; - required?: boolean; - default_value?: string; - hidden?: boolean; - query_param_key?: string; + required?: boolean | null; + default_value?: string | null; + hidden?: boolean | null; + query_param_key?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/fides__connectors__models__KeyfileCreds.ts b/clients/admin-ui/src/types/api/models/fides__connectors__models__KeyfileCreds.ts index 9a980d6228..980390f5d5 100644 --- a/clients/admin-ui/src/types/api/models/fides__connectors__models__KeyfileCreds.ts +++ b/clients/admin-ui/src/types/api/models/fides__connectors__models__KeyfileCreds.ts @@ -6,14 +6,14 @@ * The model for BigQuery credential keyfiles. */ export type fides__connectors__models__KeyfileCreds = { - type?: string; + type?: string | null; project_id: string; - private_key_id?: string; - private_key?: string; - client_email?: string; - client_id?: string; - auth_uri?: string; - token_uri?: string; - auth_provider_x509_cert_url?: string; - client_x509_cert_url?: string; + private_key_id?: string | null; + private_key?: string | null; + client_email?: string | null; + client_id?: string | null; + auth_uri?: string | null; + token_uri?: string | null; + auth_provider_x509_cert_url?: string | null; + client_x509_cert_url?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/fideslang__models__Policy.ts b/clients/admin-ui/src/types/api/models/fideslang__models__Policy.ts index 2ffa85d431..5f2b150acc 100644 --- a/clients/admin-ui/src/types/api/models/fideslang__models__Policy.ts +++ b/clients/admin-ui/src/types/api/models/fideslang__models__Policy.ts @@ -18,15 +18,15 @@ export type fideslang__models__Policy = { * Defines the Organization that this resource belongs to. */ organization_fides_key?: string; - tags?: Array; + tags?: Array | null; /** * Human-Readable name for this resource. */ - name?: string; + name?: string | null; /** * A detailed description of what this resource is. */ - description?: string; + description?: string | null; /** * * The PolicyRule resource model. diff --git a/clients/fidesui/src/components/classified-data-category-dropdown/ClassifiedDataCategoryDropdown.tsx b/clients/fidesui/src/components/classified-data-category-dropdown/ClassifiedDataCategoryDropdown.tsx index ed2b3fb74a..90a3ad2ec9 100644 --- a/clients/fidesui/src/components/classified-data-category-dropdown/ClassifiedDataCategoryDropdown.tsx +++ b/clients/fidesui/src/components/classified-data-category-dropdown/ClassifiedDataCategoryDropdown.tsx @@ -28,13 +28,13 @@ export const ClassifiedDataCategoryDropdown = ({ const mostLikelySorted = useMemo( () => mostLikelyCategories.sort((a, b) => { - if (a.confidence !== undefined && b.confidence !== undefined) { + if (a.confidence != null && b.confidence != null) { return b.confidence - a.confidence; } - if (a.confidence === undefined) { + if (a.confidence == null) { return 1; } - if (b.confidence === undefined) { + if (b.confidence == null) { return -1; } return a.fides_key.localeCompare(b.fides_key); diff --git a/clients/fidesui/src/components/classified-data-category-dropdown/types.ts b/clients/fidesui/src/components/classified-data-category-dropdown/types.ts index fbdde16bf3..722539e6c3 100644 --- a/clients/fidesui/src/components/classified-data-category-dropdown/types.ts +++ b/clients/fidesui/src/components/classified-data-category-dropdown/types.ts @@ -1,5 +1,5 @@ import { DataCategory } from "../types/api"; export interface DataCategoryWithConfidence extends DataCategory { - confidence?: number; + confidence?: number | null; } diff --git a/clients/fidesui/src/components/data-category-dropdown/helpers.ts b/clients/fidesui/src/components/data-category-dropdown/helpers.ts index b887b2f403..ed9eea3252 100644 --- a/clients/fidesui/src/components/data-category-dropdown/helpers.ts +++ b/clients/fidesui/src/components/data-category-dropdown/helpers.ts @@ -2,7 +2,7 @@ import { TaxonomyEntity, TaxonomyEntityNode } from "./types"; export const transformTaxonomyEntityToNodes = ( entities: TaxonomyEntity[], - parentKey?: string, + parentKey?: string | null, ): TaxonomyEntityNode[] => { let thisLevel: TaxonomyEntity[]; // handle the case where there are no parent keys, i.e. should just be a flat list (data subjects) diff --git a/clients/fidesui/src/components/data-category-dropdown/types.ts b/clients/fidesui/src/components/data-category-dropdown/types.ts index 57428e7e87..3775669f05 100644 --- a/clients/fidesui/src/components/data-category-dropdown/types.ts +++ b/clients/fidesui/src/components/data-category-dropdown/types.ts @@ -1,15 +1,15 @@ import { TreeNode } from "../checkbox-tree/types"; export interface TaxonomyEntityNode extends TreeNode { - description?: string; + description?: string | null; children: TaxonomyEntityNode[]; is_default: boolean; } export interface TaxonomyEntity { fides_key: string; - name?: string; - description?: string; - parent_key?: string; + name?: string | null; + description?: string | null; + parent_key?: string | null; is_default?: boolean; } diff --git a/clients/fidesui/src/components/types/api/models/DataCategory.ts b/clients/fidesui/src/components/types/api/models/DataCategory.ts index f45ba71468..f009a7591c 100644 --- a/clients/fidesui/src/components/types/api/models/DataCategory.ts +++ b/clients/fidesui/src/components/types/api/models/DataCategory.ts @@ -14,16 +14,16 @@ export type DataCategory = { * Defines the Organization that this resource belongs to. */ organization_fides_key?: string; - tags?: Array; + tags?: Array | null; /** * Human-Readable name for this resource. */ - name?: string; + name?: string | null; /** * A detailed description of what this resource is. */ - description?: string; - parent_key?: string; + description?: string | null; + parent_key?: string | null; /** * Denotes whether the resource is part of the default taxonomy or not. */ diff --git a/data/saas/config/oracle_responsys_config.yml b/data/saas/config/oracle_responsys_config.yml index 7660ed9780..819e867949 100644 --- a/data/saas/config/oracle_responsys_config.yml +++ b/data/saas/config/oracle_responsys_config.yml @@ -3,7 +3,7 @@ saas_config: name: Oracle Responsys type: oracle_responsys description: A sample schema representing the Oracle Responsys connector for Fides - version: 0.0.2 + version: 0.0.3 connector_params: - name: domain diff --git a/dev-requirements.txt b/dev-requirements.txt index a8e6c257d2..c606814938 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,10 +3,10 @@ debugpy==1.6.3 Faker==14.1.0 GitPython==3.1.41 isort==5.12.0 -mypy==0.981 +mypy==1.10.0 nox==2022.8.7 pre-commit==2.20.0 -pylint==2.15.4 +pylint==3.2.5 pytest-asyncio==0.19.0 pytest-cov==4.0.0 pytest-env==0.6.2 @@ -15,7 +15,7 @@ pytest-rerunfailures==14.0 pytest==7.2.2 requests-mock==1.10.0 setuptools>=64.0.2 -sqlalchemy-stubs +sqlalchemy-stubs==0.4 types-paramiko==3.0.0.10 types-PyYAML==6.0.11 types-redis==4.3.4 diff --git a/docs/fides/docs/development/overview.md b/docs/fides/docs/development/overview.md index 96465ac3ee..7fa8e3a710 100644 --- a/docs/fides/docs/development/overview.md +++ b/docs/fides/docs/development/overview.md @@ -23,7 +23,7 @@ The primary requirements for contributing to Fides are `Docker` and `Python`. Th _NOTE: Installing these requirements via Brew or other package managers is highly discouraged. Please use the provided links for a more stable experience._ * __Docker Desktop (version 20.10.11 or later)__ - [Docker Desktop Download Page](https://www.docker.com/products/docker-desktop/) -* __Python (version 3.8 through 3.10)__ - To simplify the installation experience and create a more stable Python installation that can be managed indepently, we recommend installing Python via Anaconda. The installer for Anaconda can be found [here](https://www.anaconda.com/download). +* __Python (version 3.9 through 3.10)__ - To simplify the installation experience and create a more stable Python installation that can be managed indepently, we recommend installing Python via Anaconda. The installer for Anaconda can be found [here](https://www.anaconda.com/download). !!! warning _Mac Users_: Apple's ARM silicon (M-series chips, i.e. M1, M2, M2 Max, etc.) have a few extra requirements to get Fides running diff --git a/noxfile.py b/noxfile.py index 565ca7bf48..e30dd11238 100644 --- a/noxfile.py +++ b/noxfile.py @@ -24,7 +24,7 @@ # pylint: enable=unused-wildcard-import, wildcard-import, wrong-import-position REQUIRED_DOCKER_VERSION = "20.10.17" -REQUIRED_PYTHON_VERSIONS = ["3.8", "3.9", "3.10"] +REQUIRED_PYTHON_VERSIONS = ["3.9", "3.10"] nox.options.sessions = ["open_docs"] diff --git a/noxfiles/docker_nox.py b/noxfiles/docker_nox.py index bd9aec3ef2..28408cbe9f 100644 --- a/noxfiles/docker_nox.py +++ b/noxfiles/docker_nox.py @@ -122,6 +122,8 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: # to get built. These images are outside of the primary `ethyca/fides` # image so some additional logic is required. if image in ("test", "prod"): + tag_name = None + if image == "prod": tag_name = get_current_tag() if image == "test": diff --git a/pyproject.toml b/pyproject.toml index 9d2f4885fd..709be9a701 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,7 @@ src_paths = ["src", "tests", "noxfiles"] disable=[ "bad-option-value", "broad-except", + "broad-exception-raised", "consider-using-f-string", "dangerous-default-value", "duplicate-code", diff --git a/requirements.txt b/requirements.txt index 0e528bb6b8..0c9628f954 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,9 +12,9 @@ dask==2022.9.2 deepdiff==6.3.0 defusedxml==0.7.1 expandvars==0.9.0 -fastapi[all]==0.89.1 -fastapi-pagination[sqlalchemy]==0.11.4 -fideslang==3.0.1 +fastapi[all]==0.111.0 +fastapi-pagination[sqlalchemy]==0.12.25 +fideslang==3.0.2 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.41 @@ -40,7 +40,8 @@ passlib[bcrypt]==1.7.4 plotly==5.13.1 pyinstrument==4.5.1 psycopg2-binary==2.9.6 -pydantic==1.10.13 +pydantic==2.7.1 +pydantic-settings==2.3.3 pydash==6.0.2 pygtrie==2.5.0 PyJWT==2.4.0 @@ -64,6 +65,6 @@ sshtunnel==0.4.0 tinycss2==1.2.1 toml==0.10.2 twilio==7.15.0 -typing_extensions==4.5.0 # pinned to work around https://github.com/pydantic/pydantic/issues/5821 +typing-extensions==4.12.2 validators==0.20.0 versioneer==0.19 diff --git a/setup.py b/setup.py index 66c79ddc6b..d6ee2d1237 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ def optional_requirements( long_description_content_type="text/markdown", url="https://github.com/ethyca/fides", entry_points={"console_scripts": ["fides=fides.cli:cli"]}, - python_requires=">=3.8, <4", + python_requires=">=3.9, <4", package_dir={"": "src"}, packages=find_packages(where="src"), include_package_data=True, @@ -75,7 +75,6 @@ def optional_requirements( classifiers=[ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Topic :: Software Development :: Libraries", diff --git a/src/fides/api/alembic/migrations/versions/216cdc7944f1_add_datasetconfig_ctl_datasets_fk.py b/src/fides/api/alembic/migrations/versions/216cdc7944f1_add_datasetconfig_ctl_datasets_fk.py index e0c26f87ab..1635ad86ca 100644 --- a/src/fides/api/alembic/migrations/versions/216cdc7944f1_add_datasetconfig_ctl_datasets_fk.py +++ b/src/fides/api/alembic/migrations/versions/216cdc7944f1_add_datasetconfig_ctl_datasets_fk.py @@ -72,7 +72,7 @@ def upgrade(): validated_dataset: Dict = Dataset( **dataset - ).dict() # Validating before we store. + ).model_dump() # Validating before we store. validated_dataset["id"] = new_ctl_dataset_id validated_dataset["fides_key"] = fides_key validated_dataset["collections"] = json.dumps(validated_dataset["collections"]) diff --git a/src/fides/api/alembic/migrations/versions/8a71872089e4_add_notice_key_to_notices.py b/src/fides/api/alembic/migrations/versions/8a71872089e4_add_notice_key_to_notices.py index f4b13c3961..678f242959 100644 --- a/src/fides/api/alembic/migrations/versions/8a71872089e4_add_notice_key_to_notices.py +++ b/src/fides/api/alembic/migrations/versions/8a71872089e4_add_notice_key_to_notices.py @@ -8,7 +8,7 @@ import sqlalchemy as sa from alembic import op -from fideslang.validation import FidesKey, FidesValidationError +from fideslang.validation import FidesKey, FidesValidationError, validate_fides_key # revision identifiers, used by Alembic. from sqlalchemy import text @@ -24,7 +24,7 @@ def validate_fides_key_suitability(names: ResultProxy, table_name: str) -> None: for row in names: name: str = row["name"].strip(" ").replace(" ", "_") try: - FidesKey.validate(name) + validate_fides_key(name) except FidesValidationError as exc: raise Exception( f"Cannot auto-migrate, adjust existing {table_name} name: '{name}' to remove invalid characters: {exc}." diff --git a/src/fides/api/alembic/migrations/versions/ffee79245c9a_add_openid_provider.py b/src/fides/api/alembic/migrations/versions/ffee79245c9a_add_openid_provider.py index 3449628827..7e247246e4 100644 --- a/src/fides/api/alembic/migrations/versions/ffee79245c9a_add_openid_provider.py +++ b/src/fides/api/alembic/migrations/versions/ffee79245c9a_add_openid_provider.py @@ -18,7 +18,6 @@ def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.create_table( "openid_provider", sa.Column("id", sa.String(length=255), nullable=False), @@ -66,12 +65,10 @@ def upgrade(): ["identifier"], unique=True, ) - # ### end Alembic commands ### def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.drop_index(op.f("ix_openid_provider_identifier"), table_name="openid_provider") op.drop_index(op.f("ix_openid_provider_id"), table_name="openid_provider") op.drop_table("openid_provider") - # ### end Alembic commands ### + op.execute("DROP TYPE providerenum") diff --git a/src/fides/api/api/v1/endpoints/config_endpoints.py b/src/fides/api/api/v1/endpoints/config_endpoints.py index 2db7536398..da1858639b 100644 --- a/src/fides/api/api/v1/endpoints/config_endpoints.py +++ b/src/fides/api/api/v1/endpoints/config_endpoints.py @@ -59,7 +59,7 @@ def patch_settings( i.e. true PATCH behavior. """ - pruned_data = data.dict(exclude_none=True) + pruned_data = data.model_dump(exclude_none=True) logger.info("PATCHing application settings") update_config: ApplicationConfig = ApplicationConfig.update_api_set(db, pruned_data) @@ -86,7 +86,7 @@ def put_settings( The record will look exactly as it is provided, i.e. true PUT behavior. """ - pruned_data = data.dict(exclude_none=True) + pruned_data = data.model_dump(exclude_none=True) logger.info("PUTing application settings") update_config: ApplicationConfig = ApplicationConfig.update_api_set( db, diff --git a/src/fides/api/api/v1/endpoints/connection_endpoints.py b/src/fides/api/api/v1/endpoints/connection_endpoints.py index 42155206e0..7368851a97 100644 --- a/src/fides/api/api/v1/endpoints/connection_endpoints.py +++ b/src/fides/api/api/v1/endpoints/connection_endpoints.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional +from typing import Annotated, Any, Dict, List, Optional from fastapi import Depends from fastapi.params import Query, Security @@ -9,7 +9,7 @@ from fastapi_pagination.ext.sqlalchemy import paginate from fideslang.validation import FidesKey from loguru import logger -from pydantic import conlist +from pydantic import Field from sqlalchemy import null, or_ from sqlalchemy.orm import Session from sqlalchemy_utils import escape_like @@ -172,7 +172,7 @@ def get_connection_detail( def patch_connections( *, db: Session = Depends(deps.get_db), - configs: conlist(CreateConnectionConfigurationWithSecrets, max_items=50), # type: ignore + configs: Annotated[List[CreateConnectionConfigurationWithSecrets], Field(max_length=50)], # type: ignore ) -> BulkPutConnectionConfiguration: """ Given a list of connection config data elements, optionally containing the secrets, @@ -205,7 +205,7 @@ def validate_and_update_secrets( ) -> TestStatusMessage: connection_config.secrets = validate_secrets( db, unvalidated_secrets, connection_config - ).dict() + ).model_dump(mode="json") # Save validated secrets, regardless of whether they've been verified. logger.info("Updating connection config secrets for '{}'", connection_key) connection_config.save(db=db) @@ -274,7 +274,7 @@ def patch_connection_config_secrets( patched_secrets = { **patched_secrets, - **unvalidated_secrets, + **unvalidated_secrets, # type: ignore[dict-item] } return validate_and_update_secrets( diff --git a/src/fides/api/api/v1/endpoints/connection_type_endpoints.py b/src/fides/api/api/v1/endpoints/connection_type_endpoints.py index bf6cf903cd..d728f7c016 100644 --- a/src/fides/api/api/v1/endpoints/connection_type_endpoints.py +++ b/src/fides/api/api/v1/endpoints/connection_type_endpoints.py @@ -4,6 +4,7 @@ from fastapi.params import Security from fastapi_pagination import Page, Params, paginate from fastapi_pagination.bases import AbstractPage +from fastapi_pagination.utils import disable_installed_extensions_check from starlette.status import HTTP_404_NOT_FOUND from fides.api.common_exceptions import NoSuchConnectionTypeSecretSchemaError @@ -25,6 +26,10 @@ V1_URL_PREFIX, ) +# FastAPI is saying that because cassandra is installed, we should use fastapi_pagination.ext.cassandra here, +# which is not relevant +disable_installed_extensions_check() + router = APIRouter(tags=["Connection Types"], prefix=V1_URL_PREFIX) diff --git a/src/fides/api/api/v1/endpoints/consent_request_endpoints.py b/src/fides/api/api/v1/endpoints/consent_request_endpoints.py index 82a27d737d..72186be3f4 100644 --- a/src/fides/api/api/v1/endpoints/consent_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/consent_request_endpoints.py @@ -403,7 +403,7 @@ def queue_privacy_request_to_propagate_consent_old_workflow( # Restrict consent preferences to just those that are executable executable_consent_preferences: List[Dict] = [ - pref.dict() + pref.model_dump(mode="json") for pref in consent_preferences.consent or [] if pref.data_use in executable_data_uses ] @@ -463,7 +463,9 @@ def set_consent_preferences( consent_request_id=consent_request_id, verification_code=data.code, ) - consent_request.preferences = [schema.dict() for schema in data.consent] + consent_request.preferences = [ + schema.model_dump(mode="json") for schema in data.consent + ] consent_request.save(db=db) if not provided_identity.hashed_value: @@ -623,6 +625,8 @@ def infer_target_identity_type( CONFIG.notifications.notification_service_type. Otherwise, the fallback order is email, phone_number, external_id, and finally fides_user_device_id. """ + target_identity_type = "" + if identity_data.email and identity_data.phone_number: messaging_method = get_messaging_method( ConfigProxy(db).notifications.notification_service_type @@ -703,7 +707,7 @@ def _prepare_consent_report( value=consent.provided_identity_id, ) consent.identity = provided_identity.as_identity_schema() # type: ignore[union-attr] - report = ConsentReport.from_orm(consent) + report = ConsentReport.model_validate(consent) return report diff --git a/src/fides/api/api/v1/endpoints/dataset_endpoints.py b/src/fides/api/api/v1/endpoints/dataset_endpoints.py index 00a07865be..9129b88240 100644 --- a/src/fides/api/api/v1/endpoints/dataset_endpoints.py +++ b/src/fides/api/api/v1/endpoints/dataset_endpoints.py @@ -1,7 +1,8 @@ -from typing import Callable, List +from typing import Annotated, Callable, List import yaml from fastapi import Depends, HTTPException, Request +from fastapi.encoders import jsonable_encoder from fastapi.params import Security from fastapi_pagination import Page, Params from fastapi_pagination.bases import AbstractPage @@ -9,8 +10,8 @@ from fideslang.models import Dataset from fideslang.validation import FidesKey from loguru import logger +from pydantic import Field from pydantic import ValidationError as PydanticValidationError -from pydantic import conlist from sqlalchemy import and_, not_, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -100,7 +101,8 @@ def validate_data_categories(dataset: Dataset, db: Session) -> None: validate_data_categories_against_db(dataset, defined_data_categories) except PydanticValidationError as e: raise HTTPException( - status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.errors() + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=jsonable_encoder(e.errors(include_url=False, include_input=False)), ) @@ -176,7 +178,7 @@ def validate_dataset( response_model=BulkPutDataset, ) def patch_dataset_configs( - dataset_pairs: conlist(DatasetConfigCtlDataset, max_items=50), # type: ignore + dataset_pairs: Annotated[List[DatasetConfigCtlDataset], Field(max_length=50)], # type: ignore db: Session = Depends(deps.get_db), connection_config: ConnectionConfig = Depends(_get_connection_config), ) -> BulkPutDataset: @@ -209,10 +211,13 @@ def patch_dataset_configs( ) try: - fetched_dataset: Dataset = Dataset.from_orm(ctl_dataset) + fetched_dataset: Dataset = Dataset.model_validate(ctl_dataset) except PydanticValidationError as e: raise HTTPException( - status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.errors() + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=jsonable_encoder( + e.errors(include_url=False, include_input=False) + ), ) validate_data_categories(fetched_dataset, db) @@ -245,7 +250,7 @@ def patch_dataset_configs( response_model=BulkPutDataset, ) def patch_datasets( - datasets: conlist(Dataset, max_items=50), # type: ignore + datasets: Annotated[List[Dataset], Field(max_length=50)], # type: ignore db: Session = Depends(deps.get_db), connection_config: ConnectionConfig = Depends(_get_connection_config), ) -> BulkPutDataset: @@ -276,7 +281,9 @@ def patch_datasets( data = { "connection_config_id": connection_config.id, "fides_key": dataset.fides_key, - "dataset": dataset.dict(), # Currently used for upserting a CTL Dataset + "dataset": dataset.model_dump( + mode="json" + ), # Currently used for upserting a CTL Dataset } create_or_update_dataset( connection_config, diff --git a/src/fides/api/api/v1/endpoints/generate.py b/src/fides/api/api/v1/endpoints/generate.py index c19f61be12..0309840edf 100644 --- a/src/fides/api/api/v1/endpoints/generate.py +++ b/src/fides/api/api/v1/endpoints/generate.py @@ -8,7 +8,7 @@ from fastapi import Depends, HTTPException, Security, status from fideslang.models import Dataset, Organization, System from loguru import logger as log -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, model_validator from sqlalchemy.ext.asyncio import AsyncSession from fides.api.api.v1.endpoints import API_PREFIX @@ -69,7 +69,7 @@ class Generate(BaseModel): target: ValidTargets type: GenerateTypes - @root_validator() + @model_validator(mode="before") @classmethod def target_matches_type(cls, values: Dict) -> Dict: """ @@ -103,7 +103,7 @@ class GenerateResponse(BaseModel): The model to house the response for generated infrastructure. """ - generate_results: Optional[List[Union[Dataset, System]]] + generate_results: Optional[List[Union[Dataset, System]]] = None @GENERATE_ROUTER.post( @@ -144,11 +144,12 @@ async def generate( All production deployments should implement HTTPS for security purposes """ - organization = await get_resource( + organization: Organization = await get_resource( sql_model_map["organization"], generate_request_payload.organization_key, db ) generate_config = generate_request_payload.generate.config generate_target = generate_request_payload.generate.target.lower() + generate_results = None try: if generate_target == "aws" and isinstance(generate_config, AWSConfig): generate_results = generate_aws( @@ -210,7 +211,7 @@ def generate_aws( log.info("Generating systems from AWS") aws_systems = generate_aws_systems(organization=organization, aws_config=aws_config) - return [i.dict(exclude_none=True) for i in aws_systems] + return [i.model_dump(exclude_none=True) for i in aws_systems] def generate_dynamodb( @@ -235,7 +236,7 @@ def generate_dynamodb( aws_config=aws_config, single_dataset=single_dataset ) - return [i.dict(exclude_none=True) for i in aws_resources] + return [i.model_dump(exclude_none=True) for i in aws_resources] async def generate_okta( @@ -258,7 +259,7 @@ async def generate_okta( okta_systems = await generate_okta_systems( organization=organization, okta_config=okta_config ) - return [i.dict(exclude_none=True) for i in okta_systems] + return [i.model_dump(exclude_none=True) for i in okta_systems] def generate_db(db_config: DatabaseConfig) -> List[Dict[str, str]]: @@ -276,7 +277,7 @@ def generate_db(db_config: DatabaseConfig) -> List[Dict[str, str]]: log.info("Generating datasets from database") db_datasets = generate_db_datasets(connection_string=db_config.connection_string) - return [i.dict(exclude_none=True) for i in db_datasets] + return [i.model_dump(exclude_none=True) for i in db_datasets] def generate_bigquery(bigquery_config: BigQueryConfig) -> List[Dict[str, str]]: @@ -291,4 +292,4 @@ def generate_bigquery(bigquery_config: BigQueryConfig) -> List[Dict[str, str]]: status_code=status.HTTP_401_UNAUTHORIZED, detail=str(error), ) - return [i.dict(exclude_none=True) for i in bigquery_datasets] + return [i.model_dump(exclude_none=True) for i in bigquery_datasets] diff --git a/src/fides/api/api/v1/endpoints/health.py b/src/fides/api/api/v1/endpoints/health.py index 365edda816..249adbe6b1 100644 --- a/src/fides/api/api/v1/endpoints/health.py +++ b/src/fides/api/api/v1/endpoints/health.py @@ -32,7 +32,7 @@ class DatabaseHealthCheck(BaseModel): """Database Healthcheck Schema""" database: str - database_revision: Optional[str] + database_revision: Optional[str] = None class WorkerHealthCheck(BaseModel): @@ -88,7 +88,7 @@ async def database_health(db: Session = Depends(get_db)) -> Dict: response = DatabaseHealthCheck( database=db_health, database_revision=revision if revision else "unknown" - ).dict() + ).model_dump(mode="json") if db_health != "healthy": raise HTTPException( @@ -130,7 +130,7 @@ async def workers_health() -> Dict: """Confirm that the API is running and healthy.""" response = WorkerHealthCheck( workers_enabled=False, workers=[], queue_counts={} - ).dict() + ).model_dump(mode="json") fides_is_using_workers = not celery_app.conf["task_always_eager"] if fides_is_using_workers: @@ -179,7 +179,7 @@ async def health() -> Dict: webserver="healthy", version=str(fides.__version__), cache=cache_health, - ).dict() + ).model_dump(mode="json") for _, value in response.items(): if value == "unhealthy": diff --git a/src/fides/api/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/api/v1/endpoints/messaging_endpoints.py index a31383d5e6..64ffc0097c 100644 --- a/src/fides/api/api/v1/endpoints/messaging_endpoints.py +++ b/src/fides/api/api/v1/endpoints/messaging_endpoints.py @@ -1,11 +1,13 @@ from typing import Dict, List, Optional from fastapi import Depends, Security +from fastapi.encoders import jsonable_encoder from fastapi_pagination import Page, Params from fastapi_pagination.bases import AbstractPage from fastapi_pagination.ext.sqlalchemy import paginate from fideslang.validation import FidesKey from loguru import logger +from pydantic import ValidationError from sqlalchemy.orm import Session from starlette.exceptions import HTTPException from starlette.status import ( @@ -272,6 +274,11 @@ def get_messaging_status( config_status=MessagingConfigStatus.not_configured, detail=f"Invalid secrets found on {messaging_config.service_type.value} messaging configuration", # type: ignore ) + except ValidationError: + return MessagingConfigStatusMessage( + config_status=MessagingConfigStatus.not_configured, + detail=f"Invalid secrets found on {messaging_config.service_type.value} messaging configuration", # type: ignore + ) return MessagingConfigStatusMessage( config_status=MessagingConfigStatus.configured, @@ -296,7 +303,7 @@ def put_default_config( "Starting upsert for default messaging config of type '{}'", messaging_config.service_type, ) - incoming_data = messaging_config.dict() + incoming_data = messaging_config.model_dump(mode="json") existing_default = MessagingConfig.get_by_type(db, messaging_config.service_type) if existing_default: # take the key of the existing default and add that to the incoming data, to ensure we overwrite the same record @@ -376,10 +383,15 @@ def update_config_secrets( status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=exc.args[0], ) - except ValueError as exc: + except ValidationError as exc: + # Remove url, input, and ctx from response. This is to prevent leaking sensitive information. + errors = exc.errors(include_url=False, include_input=False) + for err in errors: + err.pop("ctx", None) + raise HTTPException( - status_code=HTTP_400_BAD_REQUEST, - detail=exc.args[0], + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=jsonable_encoder(errors), ) logger.info( @@ -387,7 +399,7 @@ def update_config_secrets( messaging_config.key, ) try: - messaging_config.set_secrets(db=db, messaging_secrets=secrets_schema.dict()) # type: ignore + messaging_config.set_secrets(db=db, messaging_secrets=secrets_schema.model_dump(mode="json")) # type: ignore except ValueError as exc: raise HTTPException( status_code=HTTP_400_BAD_REQUEST, @@ -583,11 +595,14 @@ def update_basic_messaging_templates( ) except ValueError as e: - failed.append(BulkUpdateFailed(message=str(e), data=template)) + failed.append( + BulkUpdateFailed(message=str(e), data=template.model_dump(mode="json")) + ) except Exception: failed.append( BulkUpdateFailed( - message="Unexpected error updating template.", data=template + message="Unexpected error updating template.", + data=template.model_dump(mode="json"), ) ) diff --git a/src/fides/api/api/v1/endpoints/policy_endpoints.py b/src/fides/api/api/v1/endpoints/policy_endpoints.py index e320f091ea..5e9a66483b 100644 --- a/src/fides/api/api/v1/endpoints/policy_endpoints.py +++ b/src/fides/api/api/v1/endpoints/policy_endpoints.py @@ -1,12 +1,12 @@ -from typing import Any, Dict, List, Optional +from typing import Annotated, Any, Dict, List, Optional -from fastapi import Body, Depends, Security +from fastapi import Depends, Security from fastapi_pagination import Page, Params from fastapi_pagination.bases import AbstractPage from fastapi_pagination.ext.sqlalchemy import paginate from fideslang.validation import FidesKey from loguru import logger -from pydantic import conlist +from pydantic import Field from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from starlette.exceptions import HTTPException @@ -103,7 +103,7 @@ def create_or_update_policies( scopes=[scope_registry.POLICY_CREATE_OR_UPDATE], ), db: Session = Depends(deps.get_db), - data: conlist(schemas.Policy, max_items=50) = Body(...), # type: ignore + data: Annotated[List[schemas.Policy], Field(max_length=50)], # type: ignore ) -> schemas.BulkPutPolicyResponse: """ Given a list of policy data elements, create or update corresponding Policy objects @@ -234,7 +234,7 @@ def create_or_update_rules( ), policy_key: FidesKey, db: Session = Depends(deps.get_db), - input_data: conlist(schemas.RuleCreate, max_items=50) = Body(...), # type: ignore + input_data: Annotated[List[schemas.RuleCreate], Field(max_length=50)], # type: ignore ) -> schemas.BulkPutRuleResponse: """ Given a list of Rule data elements, create or update corresponding Rule objects @@ -282,7 +282,7 @@ def create_or_update_rules( masking_strategy_data = None if schema.masking_strategy: - masking_strategy_data = schema.masking_strategy.dict() + masking_strategy_data = schema.masking_strategy.model_dump(mode="json") try: rule = Rule.create_or_update( @@ -484,7 +484,7 @@ def create_or_update_rule_targets( policy_key: FidesKey, rule_key: FidesKey, db: Session = Depends(deps.get_db), - input_data: conlist(schemas.RuleTarget, max_items=50) = Body(...), # type: ignore + input_data: Annotated[List[schemas.RuleTarget], Field(max_length=50)], # type: ignore ) -> schemas.BulkPutRuleTargetResponse: """ Given a list of Rule data elements, create corresponding Rule objects diff --git a/src/fides/api/api/v1/endpoints/policy_webhook_endpoints.py b/src/fides/api/api/v1/endpoints/policy_webhook_endpoints.py index be8cbcf538..f1e5bc35f5 100644 --- a/src/fides/api/api/v1/endpoints/policy_webhook_endpoints.py +++ b/src/fides/api/api/v1/endpoints/policy_webhook_endpoints.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Annotated, List from fastapi import Body, Depends, Security from fastapi_pagination import Page, Params @@ -6,7 +6,7 @@ from fastapi_pagination.ext.sqlalchemy import paginate from fideslang.validation import FidesKey from loguru import logger -from pydantic import conlist +from pydantic import Field from sqlalchemy.orm import Session from starlette.exceptions import HTTPException from starlette.status import HTTP_200_OK, HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND @@ -100,7 +100,7 @@ def put_webhooks( policy = get_policy_or_error(db, policy_key) keys = [ - get_key_from_data(webhook.dict(), type(webhook_cls).__name__) + get_key_from_data(webhook.model_dump(mode="json"), type(webhook_cls).__name__) for webhook in webhooks ] names = [webhook.name for webhook in webhooks] @@ -174,7 +174,7 @@ def create_or_update_pre_execution_webhooks( *, policy_key: FidesKey, db: Session = Depends(deps.get_db), - webhooks: conlist(schemas.PolicyWebhookCreate, max_items=50) = Body(...), # type: ignore + webhooks: Annotated[List[schemas.PolicyWebhookCreate], Field(max_length=50)], # type: ignore ) -> List[PolicyPreWebhook]: """ Create or update the list of Policy Pre-Execution Webhooks that run **before** query execution. @@ -197,7 +197,7 @@ def create_or_update_post_execution_webhooks( *, policy_key: FidesKey, db: Session = Depends(deps.get_db), - webhooks: conlist(schemas.PolicyWebhookCreate, max_items=50) = Body(...), # type: ignore + webhooks: Annotated[List[schemas.PolicyWebhookCreate], Field(max_length=50)], # type: ignore ) -> List[PolicyPostWebhook]: """ Create or update the list of Policy Post-Execution Webhooks that run **after** query execution. @@ -290,7 +290,7 @@ def _patch_webhook( """ policy = get_policy_or_error(db, policy_key) loaded_webhook = get_policy_webhook_or_error(db, policy, webhook_key, webhook_cls) - data = webhook_body.dict(exclude_none=True) + data = webhook_body.model_dump(exclude_none=True) if data.get("connection_config_key"): connection_config = get_connection_config_or_error( diff --git a/src/fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py b/src/fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py index 5c8e5a5878..81d53b6a4d 100644 --- a/src/fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py +++ b/src/fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Annotated, List from fastapi import Body, Depends, Security from fastapi_pagination import Page, Params @@ -6,7 +6,7 @@ from fastapi_pagination.ext.sqlalchemy import paginate from fideslang.validation import FidesKey from loguru import logger -from pydantic import conlist +from pydantic import Field from sqlalchemy.orm import Session from starlette.exceptions import HTTPException from starlette.status import HTTP_200_OK, HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND @@ -91,7 +91,7 @@ def get_pre_approval_webhook_detail( def create_or_update_pre_execution_webhooks( *, db: Session = Depends(deps.get_db), - webhooks: conlist(schemas.PreApprovalWebhookCreate, max_items=50) = Body(...), # type: ignore + webhooks: Annotated[List[schemas.PreApprovalWebhookCreate], Field(max_length=50)], # type: ignore ) -> List[PreApprovalWebhook]: """ Create or update the list of Pre-Approval Webhooks that run as soon as a request is created. @@ -102,7 +102,7 @@ def create_or_update_pre_execution_webhooks( pre_approval_webhooks = PreApprovalWebhook.query(db=db) keys = [ - get_key_from_data(webhook.dict(), PreApprovalWebhook.__name__) + get_key_from_data(webhook.model_dump(mode="json"), PreApprovalWebhook.__name__) for webhook in webhooks ] names = [webhook.name for webhook in webhooks] @@ -171,7 +171,7 @@ def update_pre_execution_webhook( """PATCH a single Pre-Approval Webhook that runs as soon as Privacy Request is created.""" loaded_webhook = get_pre_approval_webhook_or_error(db, webhook_key) - data = webhook_body.dict(exclude_none=True) + data = webhook_body.model_dump(exclude_none=True) if data.get("connection_config_key"): connection_config = get_connection_config_or_error( diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index 2565bf01cc..8deb516e31 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -4,17 +4,29 @@ import io from collections import defaultdict from datetime import datetime -from typing import Any, Callable, DefaultDict, Dict, List, Literal, Optional, Set, Union +from typing import ( + Annotated, + Any, + Callable, + DefaultDict, + Dict, + List, + Literal, + Optional, + Set, + Union, +) import sqlalchemy from fastapi import Body, Depends, HTTPException, Security +from fastapi.encoders import jsonable_encoder from fastapi.params import Query as FastAPIQuery from fastapi_pagination import Page, Params from fastapi_pagination.bases import AbstractPage from fastapi_pagination.ext.sqlalchemy import paginate from loguru import logger +from pydantic import Field from pydantic import ValidationError as PydanticValidationError -from pydantic import conlist from sqlalchemy import cast, column, null, or_, select from sqlalchemy.orm import Query, Session from sqlalchemy.sql.expression import nullslast @@ -199,7 +211,7 @@ def create_privacy_request( *, db: Session = Depends(deps.get_db), config_proxy: ConfigProxy = Depends(deps.get_config_proxy), - data: conlist(PrivacyRequestCreate, max_items=50) = Body(...), # type: ignore + data: Annotated[List[PrivacyRequestCreate], Field(max_length=50)], # type: ignore ) -> BulkPostPrivacyRequests: """ Given a list of privacy request data elements, create corresponding PrivacyRequest objects @@ -219,7 +231,7 @@ def create_privacy_request_authenticated( *, db: Session = Depends(deps.get_db), config_proxy: ConfigProxy = Depends(deps.get_config_proxy), - data: conlist(PrivacyRequestCreate, max_items=50) = Body(...), # type: ignore + data: Annotated[List[PrivacyRequestCreate], Field(max_length=50)], # type: ignore ) -> BulkPostPrivacyRequests: """ Given a list of privacy request data elements, create corresponding PrivacyRequest objects @@ -262,9 +274,9 @@ def _send_privacy_request_receipt_message_to_user( "message_meta": FidesopsMessage( action_type=MessagingActionType.PRIVACY_REQUEST_RECEIPT, body_params=RequestReceiptBodyParams(request_types=request_types), - ).dict(), + ).model_dump(mode="json"), "service_type": service_type, - "to_identity": to_identity.dict(), + "to_identity": to_identity.model_dump(mode="json"), "property_id": property_id, }, ) @@ -310,7 +322,7 @@ def privacy_request_csv_download( [ pr.status.value if pr.status else None, pr.policy.rules[0].action_type if len(pr.policy.rules) > 0 else None, - pr.get_persisted_identity().dict(), + pr.get_persisted_identity().model_dump(mode="json"), pr.get_persisted_custom_privacy_request_fields(), pr.created_at, pr.reviewed_by, @@ -354,7 +366,9 @@ def execution_and_audit_logs_by_dataset_name( ExecutionLog.created_at, ExecutionLog.updated_at, ExecutionLog.message, - cast(ExecutionLog.status, sqlalchemy.String).label("status"), + cast(ExecutionLog.status, sqlalchemy.String).label( + "status" + ), # Casting to string so we can perform a union of execution log and audit log statuses ExecutionLog.privacy_request_id, ExecutionLog.dataset_name, ExecutionLog.collection_name, @@ -369,7 +383,9 @@ def execution_and_audit_logs_by_dataset_name( AuditLog.created_at, AuditLog.updated_at, AuditLog.message, - cast(AuditLog.action.label("status"), sqlalchemy.String).label("status"), + cast(AuditLog.action.label("status"), sqlalchemy.String).label( + "status" + ), # Casting to string so we can perform a union of execution log and audit log statuses AuditLog.privacy_request_id, null().label("dataset_name"), null().label("collection_name"), @@ -1183,7 +1199,9 @@ def review_privacy_request( failed.append( { "message": "Cannot transition status", - "data": PrivacyRequestResponse.from_orm(privacy_request), + "data": PrivacyRequestResponse.model_validate( + privacy_request + ).model_dump(mode="json"), } ) continue @@ -1195,7 +1213,9 @@ def review_privacy_request( except Exception: failure = { "message": "Privacy request could not be updated", - "data": PrivacyRequestResponse.from_orm(privacy_request), + "data": PrivacyRequestResponse.model_validate( + privacy_request + ).model_dump(mode="json"), } failed.append(failure) else: @@ -1236,9 +1256,9 @@ def _send_privacy_request_review_message_to_user( if action_type is MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY else None ), - ).dict(), + ).model_dump(mode="json"), "service_type": service_type, - "to_identity": to_identity.dict(), + "to_identity": to_identity.model_dump(mode="json"), "property_id": property_id, }, ) @@ -1643,7 +1663,8 @@ def _handle_manual_webhook_input( ) except PydanticValidationError as exc: raise HTTPException( - status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=exc.errors() + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=jsonable_encoder(exc.errors(include_url=False, include_input=False)), ) logger.info( @@ -1732,7 +1753,9 @@ def privacy_request_data_transfer( detail=f"No privacy request with id {privacy_request_id} found", ) - rule = Rule.filter(db=db, conditions=(Rule.key == rule_key)).first() + rule = Rule.filter( + db=db, conditions=(Rule.key == rule_key) + ).first() # pylint: disable=superfluous-parens if not rule: raise HTTPException( status_code=HTTP_404_NOT_FOUND, @@ -1964,7 +1987,7 @@ def resume_privacy_request_from_requires_input( def create_privacy_request_func( db: Session, config_proxy: ConfigProxy, - data: conlist(PrivacyRequestCreate), # type: ignore + data: Annotated[List[PrivacyRequestCreate], Field()], # type: ignore authenticated: bool = False, privacy_preferences: List[ PrivacyPreferenceHistory @@ -1995,13 +2018,13 @@ def create_privacy_request_func( "property_id", ] for privacy_request_data in data: - if not any(privacy_request_data.identity.dict().values()): + if not any(privacy_request_data.identity.model_dump(mode="json").values()): logger.warning( "Create failed for privacy request with no identity provided" ) failure = { "message": "You must provide at least one identity to process", - "data": privacy_request_data, + "data": privacy_request_data.model_dump(mode="json"), } failed.append(failure) continue @@ -2016,7 +2039,7 @@ def create_privacy_request_func( ) failure = { "message": "Property id must be valid to process", - "data": privacy_request_data, + "data": privacy_request_data.model_dump(mode="json"), } failed.append(failure) continue @@ -2035,7 +2058,7 @@ def create_privacy_request_func( failure = { "message": f"Policy with key {privacy_request_data.policy_key} does not exist", - "data": privacy_request_data, + "data": privacy_request_data.model_dump(mode="json"), } failed.append(failure) continue @@ -2050,7 +2073,7 @@ def create_privacy_request_func( attr = getattr(privacy_request_data, field) if attr is not None: if field == "consent_preferences": - attr = [consent.dict() for consent in attr] + attr = [consent.model_dump(mode="json") for consent in attr] kwargs[field] = attr diff --git a/src/fides/api/api/v1/endpoints/registration_endpoints.py b/src/fides/api/api/v1/endpoints/registration_endpoints.py index 73f374cc0f..450fc298be 100644 --- a/src/fides/api/api/v1/endpoints/registration_endpoints.py +++ b/src/fides/api/api/v1/endpoints/registration_endpoints.py @@ -77,7 +77,7 @@ async def update_registration_status( ) registration, _ = UserRegistration.create_or_update( # type: ignore[assignment] db=db, - data=data.dict(), + data=data.model_dump(mode="json"), ) if send_to_fideslog: await send_registration(registration) diff --git a/src/fides/api/api/v1/endpoints/router_factory.py b/src/fides/api/api/v1/endpoints/router_factory.py index 8ecc5d00b3..1a5e34ce29 100644 --- a/src/fides/api/api/v1/endpoints/router_factory.py +++ b/src/fides/api/api/v1/endpoints/router_factory.py @@ -4,10 +4,10 @@ Mostly used for `ctl`-related objects. """ - from typing import Dict, List from fastapi import Depends, HTTPException, Response, Security, status +from fastapi.encoders import jsonable_encoder from fideslang import FidesModelType from fideslang.models import Dataset from fideslang.validation import FidesKey @@ -63,7 +63,8 @@ async def validate_data_categories( validate_data_categories_against_db(dataset, defined_data_categories) except PydanticValidationError as e: raise HTTPException( - status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.errors() + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=jsonable_encoder(e.errors(include_url=False, include_input=False)), ) @@ -148,7 +149,7 @@ async def create( raise errors.ForbiddenIsDefaultTaxonomyError( model_type, resource.fides_key, action="create" ) - return await create_resource(sql_model, resource.dict(), db) + return await create_resource(sql_model, resource.model_dump(mode="json"), db) return router @@ -249,7 +250,7 @@ async def update( if isinstance(resource, Dataset): await validate_data_categories(resource, db) await forbid_if_editing_is_default(sql_model, resource.fides_key, resource, db) - return await update_resource(sql_model, resource.dict(), db) + return await update_resource(sql_model, resource.model_dump(mode="json"), db) return router @@ -325,7 +326,7 @@ async def upsert( """ sql_model = sql_model_map[model_type] - resource_dicts = [resource.dict() for resource in resources] + resource_dicts = [resource.model_dump(mode="json") for resource in resources] for resource in resources: if isinstance(resource, Dataset): await validate_data_categories(resource, db) @@ -388,7 +389,9 @@ async def delete( await forbid_if_default(sql_model, fides_key, db) deleted_resource = await delete_resource(sql_model, fides_key, db) # Convert the resource to a dict explicitly for the response - deleted_resource_dict = fides_model.from_orm(deleted_resource).dict() + deleted_resource_dict = fides_model.model_validate(deleted_resource).model_dump( + mode="json" + ) return { "message": "resource deleted", "resource": deleted_resource_dict, diff --git a/src/fides/api/api/v1/endpoints/saas_config_endpoints.py b/src/fides/api/api/v1/endpoints/saas_config_endpoints.py index 7199f41d3e..4e8d1b57e9 100644 --- a/src/fides/api/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fides/api/api/v1/endpoints/saas_config_endpoints.py @@ -337,7 +337,7 @@ def instantiate_connection( connection_config.secrets = validate_secrets( db, template_values.secrets, connection_config - ).dict() + ).model_dump(mode="json") if system: connection_config.system_id = system.id connection_config.save(db=db) # Not persisted to db until secrets are validated diff --git a/src/fides/api/api/v1/endpoints/storage_endpoints.py b/src/fides/api/api/v1/endpoints/storage_endpoints.py index b1312d7791..e4916e4e1c 100644 --- a/src/fides/api/api/v1/endpoints/storage_endpoints.py +++ b/src/fides/api/api/v1/endpoints/storage_endpoints.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import Annotated, Dict, List, Optional from fastapi import Body, Depends, Security from fastapi_pagination import Page, Params @@ -6,7 +6,7 @@ from fastapi_pagination.ext.sqlalchemy import paginate from fideslang.validation import FidesKey from loguru import logger -from pydantic import conlist +from pydantic import Field from requests import RequestException from sqlalchemy.orm import Session from starlette.exceptions import HTTPException @@ -122,7 +122,7 @@ def upload_data( def patch_config( *, db: Session = Depends(deps.get_db), - storage_configs: conlist(StorageDestination, max_items=50), # type: ignore + storage_configs: Annotated[List[StorageDestination], Field(max_length=50)], # type: ignore ) -> BulkPutStorageConfigResponse: """ Given a list of storage destination elements, create or update corresponding StorageConfig objects @@ -135,7 +135,7 @@ def patch_config( for destination in storage_configs: try: storage_config = StorageConfig.create_or_update( - db=db, data=destination.dict() + db=db, data=destination.model_dump(mode="json") ) except KeyOrNameAlreadyExists as exc: logger.warning( @@ -145,7 +145,7 @@ def patch_config( ) failure = { "message": exc.args[0], - "data": destination.dict(), + "data": destination.model_dump(mode="json"), } failed.append(BulkUpdateFailed(**failure)) continue @@ -159,7 +159,7 @@ def patch_config( BulkUpdateFailed( **{ "message": "Error creating or updating storage config.", - "data": destination.dict(), + "data": destination.model_dump(mode="json"), } ) ) @@ -210,7 +210,7 @@ def put_config_secrets( logger.info("Updating storage config secrets for config with key '{}'", config_key) try: - storage_config.set_secrets(db=db, storage_secrets=secrets_schema.dict()) # type: ignore + storage_config.set_secrets(db=db, storage_secrets=secrets_schema.model_dump(mode="json")) # type: ignore except ValueError as exc: raise HTTPException( status_code=HTTP_400_BAD_REQUEST, @@ -442,7 +442,7 @@ def put_default_config( "Starting upsert for default storage of type '{}'", incoming_storage_config.type ) - incoming_data = incoming_storage_config.dict() + incoming_data = incoming_storage_config.model_dump(mode="json") existing_default = get_default_storage_config_by_type( db, incoming_storage_config.type ) @@ -524,7 +524,7 @@ def put_default_config_secrets( storage_type.value, ) try: - storage_config.set_secrets(db=db, storage_secrets=secrets_schema.dict()) # type: ignore + storage_config.set_secrets(db=db, storage_secrets=secrets_schema.model_dump(mode="json")) # type: ignore except ValueError as exc: raise HTTPException( status_code=HTTP_400_BAD_REQUEST, diff --git a/src/fides/api/api/v1/endpoints/system.py b/src/fides/api/api/v1/endpoints/system.py index df21947348..c0cf954b92 100644 --- a/src/fides/api/api/v1/endpoints/system.py +++ b/src/fides/api/api/v1/endpoints/system.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional, Union +from typing import Annotated, Dict, List, Optional, Union from fastapi import Depends, HTTPException, Query, Response, Security from fastapi_pagination import Page, Params @@ -8,7 +8,7 @@ from fideslang.models import System as SystemSchema from fideslang.validation import FidesKey from loguru import logger -from pydantic.types import conlist +from pydantic import Field from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.orm import Session @@ -126,7 +126,7 @@ def get_system_connections( ) def patch_connections( fides_key: str, - configs: conlist(CreateConnectionConfigurationWithSecrets, max_items=50), # type: ignore + configs: Annotated[List[CreateConnectionConfigurationWithSecrets], Field(max_length=50)], # type: ignore db: Session = Depends(deps.get_db), ) -> BulkPutConnectionConfiguration: """ @@ -174,13 +174,16 @@ def patch_connection_secrets( if connection_config.secrets is not None: for key, value in connection_config.secrets.items(): if key not in unvalidated_secrets: + # unvalidated_secrets is actually a dictionary here. connection_secrets_schemas + # are just provided for documentation but the data was not parsed up front. + # That happens below in validate_secrets. unvalidated_secrets[key] = value # type: ignore else: connection_config.secrets = {} validated_secrets = validate_secrets( db, unvalidated_secrets, connection_config - ).dict() + ).model_dump(mode="json") for key, value in validated_secrets.items(): connection_config.secrets[key] = value # type: ignore @@ -335,7 +338,9 @@ async def delete( async with db.begin(): await db.delete(system_to_delete) # Convert the resource to a dict explicitly for the response - deleted_resource_dict = SystemSchema.from_orm(system_to_delete).dict() + deleted_resource_dict = SystemSchema.model_validate(system_to_delete).model_dump( + mode="json" + ) return { "message": "resource deleted", "resource": deleted_resource_dict, diff --git a/src/fides/api/api/v1/endpoints/user_endpoints.py b/src/fides/api/api/v1/endpoints/user_endpoints.py index 6b75115743..488a9bb831 100644 --- a/src/fides/api/api/v1/endpoints/user_endpoints.py +++ b/src/fides/api/api/v1/endpoints/user_endpoints.py @@ -134,7 +134,7 @@ async def update_user( db=db, ) - user.update(db=db, data=data.dict()) + user.update(db=db, data=data.model_dump(mode="json")) logger.info("Updated user with id: '{}'.", user.id) return user @@ -450,7 +450,7 @@ def create_user( detail="User with this email address already exists.", ) - user = FidesUser.create(db=db, data=user_data.dict()) + user = FidesUser.create(db=db, data=user_data.model_dump(mode="json")) # invite user via email invite_user(db=db, config_proxy=config_proxy, user=user) diff --git a/src/fides/api/api/v1/endpoints/user_permission_endpoints.py b/src/fides/api/api/v1/endpoints/user_permission_endpoints.py index 16bb72f9db..4a039c4f13 100644 --- a/src/fides/api/api/v1/endpoints/user_permission_endpoints.py +++ b/src/fides/api/api/v1/endpoints/user_permission_endpoints.py @@ -79,10 +79,10 @@ async def create_user_permissions( await owner_role_permission_check(db, permissions.roles, authorization) if user.client: # Just in case - this shouldn't happen in practice. - user.client.update(db=db, data=permissions.dict()) + user.client.update(db=db, data=permissions.model_dump(mode="json")) logger.info("Created FidesUserPermission record") return FidesUserPermissions.create( - db=db, data={"user_id": user_id, **permissions.dict()} + db=db, data={"user_id": user_id, **permissions.model_dump(mode="json")} ) diff --git a/src/fides/api/app_setup.py b/src/fides/api/app_setup.py index ea80c5c236..ac2cd2abfe 100644 --- a/src/fides/api/app_setup.py +++ b/src/fides/api/app_setup.py @@ -4,7 +4,7 @@ # pylint: disable=too-many-branches from logging import DEBUG -from typing import List +from typing import AsyncGenerator, List from fastapi import APIRouter, FastAPI from fastapi.routing import APIRoute @@ -63,6 +63,7 @@ def create_fides_app( + lifespan: AsyncGenerator[None, None], routers: List = ROUTERS, app_version: str = VERSION, security_env: str = CONFIG.security.env, @@ -73,11 +74,13 @@ def create_fides_app( "Logger configuration options in use" ) - fastapi_app = FastAPI(title="fides", version=app_version) + fastapi_app = FastAPI(title="fides", version=app_version, lifespan=lifespan, separate_input_output_schemas=False) # type: ignore fastapi_app.state.limiter = fides_limiter - fastapi_app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) + # Starlette bug causing this to fail mypy + fastapi_app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) # type: ignore for handler in ExceptionHandlers.get_handlers(): - fastapi_app.add_exception_handler(FunctionalityNotConfigured, handler) + # Starlette bug causing this to fail mypy + fastapi_app.add_exception_handler(FunctionalityNotConfigured, handler) # type: ignore fastapi_app.add_middleware(SlowAPIMiddleware) for router in routers: @@ -184,8 +187,8 @@ async def run_database_startup(app: FastAPI) -> None: try: ConfigProxy(db).load_current_cors_domains_into_middleware(app) except Exception as e: - logger.error("Error occured while loading CORS domains: {}", str(e)) - raise FidesError(f"Error occured while loading CORS domains: {str(e)}") + logger.error("Error occurred while loading CORS domains: {}", str(e)) + raise FidesError(f"Error occurred while loading CORS domains: {str(e)}") finally: db.close() @@ -217,8 +220,8 @@ def check_redis() -> None: except (RedisConnectionError, RedisError, ResponseError) as e: logger.error("Connection to cache failed: {}", str(e)) return - else: - logger.debug("Connection to cache succeeded") + + logger.debug("Connection to cache succeeded") def load_tcf_purpose_overrides() -> None: diff --git a/src/fides/api/custom_types.py b/src/fides/api/custom_types.py index f41910189e..b5906b46aa 100644 --- a/src/fides/api/custom_types.py +++ b/src/fides/api/custom_types.py @@ -2,48 +2,44 @@ """Logic related to sanitizing and validating user application input.""" from html import escape from re import compile as regex -from typing import Any, Generator +from typing import Annotated from nh3 import clean -from pydantic import AnyUrl, BaseConfig -from pydantic.fields import ModelField +from pydantic import AfterValidator, AnyHttpUrl, AnyUrl, BeforeValidator from fides.api.util.unsafe_file_util import verify_css -class SafeStr(str): +def validate_safe_str(val: str) -> str: """ - This class is designed to be used in place of the `str` type + Validator for SafeStr that is designed to be used in place of the `str` type any place where user input is expected. The validation applied here does the typical sanitization/cleanup that is required when dealing with user-supplied input. """ + # HTML Escapes + value = escape(val) - @classmethod - def __get_validators__(cls) -> Generator: # pragma: no cover - yield cls.validate + if len(value) > 32000: + raise ValueError("Value must be 32000 characters or less.") - @classmethod - def validate(cls, value: str) -> str: - # HTML Escapes - value = escape(value) + return value - if len(value) > 32000: - raise ValueError("Value must be 32000 characters or less.") - return value +SafeStr = Annotated[str, BeforeValidator(validate_safe_str)] -class HtmlStr(str): +def validate_html_str(val: str) -> str: """ - This class is designed to be used in place of the `str` type + Validator function for HTMLStr - designed to be used in place of the `str` type any place where user inputted HTML text is expected. The validation applied here enforces that only a subset of "safe" HTML is supported to prevent XSS or similar attacks. - """ + """ + # Assert text doesn't include an complex/malicious HTML. # Allow only basic markup tags, for extra safety ALLOWED_HTML_TAGS = { "a", @@ -70,89 +66,72 @@ class HtmlStr(str): "strong", "u", } + if val: + return clean(val, tags=ALLOWED_HTML_TAGS) + return val - @classmethod - def __get_validators__(cls) -> Generator: # pragma: no cover - yield cls.validate - @classmethod - def validate(cls, value: str) -> str: - """Assert text doesn't include an complex/malicious HTML.""" - if value: - return clean(value, tags=cls.ALLOWED_HTML_TAGS) - return value +HtmlStr = Annotated[str, BeforeValidator(validate_html_str)] -class PhoneNumber(str): - """ - Format validated type for phone numbers. +def validate_phone_number(value: str) -> str: + """Validator for PhoneNumber Standard format can be found here: https://en.wikipedia.org/wiki/E.164 """ + # The front-end sends an empty string if the user doesn't input anything + if value == "": + return "" + max_length = 16 # Includes the + + min_length = 9 + pattern = regex(r"^\+[1-9]\d{1,14}$") + if len(value) > max_length or len(value) < min_length or not pattern.search(value): + raise ValueError( + "Phone number must be formatted in E.164 format, i.e. '+15558675309'." + ) + return value + + +PhoneNumber = Annotated[str, BeforeValidator(validate_phone_number)] + - @classmethod - def __get_validators__(cls) -> Generator: - yield cls.validate - - @classmethod - def validate(cls, value: str) -> str: - # The front-end sends an empty string if the user doesn't input anything - if value == "": - return "" - max_length = 16 # Includes the + - min_length = 9 - pattern = regex(r"^\+[1-9]\d{1,14}$") - if ( - len(value) > max_length - or len(value) < min_length - or not pattern.search(value) - ): - raise ValueError( - "Phone number must be formatted in E.164 format, i.e. '+15558675309'." - ) - return value - - -class GPPMechanismConsentValue(str): +def validate_gpp_mechanism_consent_value(value: str) -> str: """ - Allowable consent values for GPP Mechanism Mappings. + Validator for GPPMechanismConsentValue - Allowable consent values for GPP Mechanism Mappings. """ + pattern = regex(r"^\d+$") + if not isinstance(value, str) or not pattern.search(value): + raise ValueError("GPP Mechanism consent value must be a string of digits.") + return value - @classmethod - def __get_validators__(cls) -> Generator: - yield cls.validate - @classmethod - def validate(cls, value: str) -> str: - pattern = regex(r"^\d+$") - if not isinstance(value, str) or not pattern.search(value): - raise ValueError("GPP Mechanism consent value must be a string of digits.") - return value +GPPMechanismConsentValue = Annotated[ + str, BeforeValidator(validate_gpp_mechanism_consent_value) +] -class URLOrigin(AnyUrl): +def validate_path_of_url(value: AnyUrl) -> str: """ A URL origin value. See https://developer.mozilla.org/en-US/docs/Glossary/Origin. We perform the basic URL validation, but also prevent URLs with paths, as paths are not part of an origin. """ + if value.path and value.path != "/": + raise ValueError("URL origin values cannot contain a path.") - @classmethod - def __get_validators__(cls) -> Generator: - yield cls.validate + # Intentionally serializing as a string instead of a URL. + # Intentionally removing trailing slash. In Pydantic V2 AnyURL will add a trailing slash to root URL's so stripping this off. + return str(value).rstrip("/") - @classmethod - def validate(cls, value: Any, field: ModelField, config: BaseConfig) -> AnyUrl: - value = super().validate(value, field, config) - if value.path: - raise ValueError("URL origin values cannot contain a path.") - return value +URLOriginString = Annotated[AnyUrl, AfterValidator(validate_path_of_url)] -class CssStr(str): +def validate_css_str(value: str) -> str: """ + Validator for CssStr + A custom string type that represents a valid CSS stylesheet. The `CssStr` type automatically validates the input value using the `verify_css` function @@ -161,14 +140,20 @@ class CssStr(str): If the input value is not a string or fails the CSS validation, a `ValidationError` is raised. """ + if not isinstance(value, str): + raise TypeError("CssStr must be a string") + verify_css(value) + return value + + +CssStr = Annotated[str, BeforeValidator(validate_css_str)] + + +def validate_path_of_http_url_no_slash(value: AnyHttpUrl) -> str: + """Converts an AnyHttpUrl to a string and strips trailing slash""" + return str(value).rstrip("/") - @classmethod - def __get_validators__(cls) -> Generator: - yield cls.validate - @classmethod - def validate(cls, value: Any) -> str: - if not isinstance(value, str): - raise TypeError("CssStr must be a string") - verify_css(value) - return value +AnyHttpUrlStringRemovesSlash = Annotated[ + AnyHttpUrl, AfterValidator(validate_path_of_http_url_no_slash) +] diff --git a/src/fides/api/db/base_class.py b/src/fides/api/db/base_class.py index b5b91300c8..ed9e8e9732 100644 --- a/src/fides/api/db/base_class.py +++ b/src/fides/api/db/base_class.py @@ -7,7 +7,7 @@ from typing import Any, Optional, Type, TypeVar from uuid import uuid4 -from fideslang.models import FidesKey # type: ignore +from fideslang.validation import validate_fides_key from sqlalchemy import Column, DateTime, String from sqlalchemy.ext.declarative import declarative_base, declared_attr from sqlalchemy.ext.mutable import MutableDict @@ -57,12 +57,16 @@ def get_key_from_data(data: dict[str, Any], cls_name: str) -> str: If no key, uses a snake-cased name. Will be used as the URL onupdate applicable classes. """ - key = FidesKey.validate(data.get("key")) if data.get("key") else None + supplied_key = data.get("key") + if supplied_key and not isinstance(supplied_key, str): # Mypy check + raise KeyValidationError("Supplied key must be a string") + + key = validate_fides_key(supplied_key) if supplied_key else None if key is None: name = data.get("name") if name is None: raise KeyValidationError(f"{cls_name} requires a name.") - key = FidesKey.validate(to_snake_case(name)) + key = validate_fides_key(to_snake_case(name)) return key diff --git a/src/fides/api/db/crud.py b/src/fides/api/db/crud.py index 2236b166fe..2cf95e9b5d 100644 --- a/src/fides/api/db/crud.py +++ b/src/fides/api/db/crud.py @@ -4,7 +4,7 @@ """ from collections import defaultdict -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, Type, TypeVar from fastapi import HTTPException from loguru import logger as log @@ -19,19 +19,22 @@ from sqlalchemy.sql import Select from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY -from fides.api.db.base import Base # type: ignore[attr-defined] from fides.api.models.sql_models import ( # type: ignore[attr-defined] CustomField, CustomFieldDefinition, + FidesBase, ResourceTypes, ) from fides.api.util import errors +# Helps return type be linked to the type of the parameter +T = TypeVar("T", bound="FidesBase") + # CRUD Functions async def create_resource( - sql_model: Base, resource_dict: Dict, async_session: AsyncSession -) -> Base: + sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession +) -> T: """Create a resource in the database.""" with log.contextualize( sql_model=sql_model.__name__, fides_key=resource_dict["fides_key"] @@ -53,7 +56,7 @@ async def create_resource( async with async_session.begin(): try: log.debug("Creating resource") - query = _insert(sql_model).values(resource_dict) + query = _insert(sql_model.__table__).values(resource_dict) await async_session.execute(query) except SQLAlchemyError: sa_error = errors.QueryError() @@ -68,7 +71,7 @@ async def create_resource( async def get_custom_fields_filtered( async_session: AsyncSession, resource_types_to_ids: Dict[ResourceTypes, List[str]] = defaultdict(list), -) -> Base: +) -> FidesBase: """ Utility function to construct a filtered query for custom field values based on provided mapping of resource types to resource IDs. @@ -113,11 +116,11 @@ async def get_custom_fields_filtered( async def get_resource( - sql_model: Base, + sql_model: Type[T], fides_key: str, async_session: AsyncSession, raise_not_found: bool = True, -) -> Base: +) -> T: """ Get a resource from the database by its FidesKey. @@ -146,13 +149,13 @@ async def get_resource( async def get_resource_with_custom_fields( - sql_model: Base, fides_key: str, async_session: AsyncSession + sql_model: Type[T], fides_key: str, async_session: AsyncSession ) -> Dict[str, Any]: """Get a resource from the databse by its FidesKey including it's custom fields. Returns a dictionary of that resource. """ - resource = await get_resource(sql_model, fides_key, async_session) + resource: T = await get_resource(sql_model, fides_key, async_session) resource_dict = resource.__dict__ resource_dict.pop("_sa_instance_state", None) @@ -198,7 +201,7 @@ async def get_resource_with_custom_fields( return resource_dict -async def list_resource(sql_model: Base, async_session: AsyncSession) -> List[Base]: +async def list_resource(sql_model: Type[T], async_session: AsyncSession) -> List[T]: """ Get a list of all of the resources of this type from the database. @@ -209,8 +212,8 @@ async def list_resource(sql_model: Base, async_session: AsyncSession) -> List[Ba async def list_resource_query( - async_session: AsyncSession, query: Select, sql_model: Base = Base -) -> List[Base]: + async_session: AsyncSession, query: Select, sql_model: Type[T] +) -> List[T]: """ Utility function to wrap a select query in generic "list_resource" execution handling. Wrapping includes execution against the DB session, logging and error handling. @@ -233,7 +236,7 @@ async def list_resource_query( async def update_resource( - sql_model: Base, resource_dict: Dict, async_session: AsyncSession + sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession ) -> Dict: """Update a resource in the database by its fides_key.""" @@ -246,7 +249,7 @@ async def update_resource( try: log.debug("Updating resource") await async_session.execute( - _update(sql_model) + _update(sql_model.__table__) .where(sql_model.fides_key == resource_dict["fides_key"]) .values(resource_dict) ) @@ -261,7 +264,7 @@ async def update_resource( async def upsert_resources( - sql_model: Base, resource_dicts: List[Dict], async_session: AsyncSession + sql_model: Type[T], resource_dicts: List[Dict], async_session: AsyncSession ) -> Tuple[int, int]: """ Insert new resources into the database. If a resource already exists, @@ -278,7 +281,7 @@ async def upsert_resources( try: log.debug("Upserting resources") insert_stmt = ( - _insert(sql_model) + _insert(sql_model.__table__) .values(resource_dicts) .returning( (column("xmax") == 0), # Row was inserted @@ -312,8 +315,8 @@ async def upsert_resources( async def delete_resource( - sql_model: Base, fides_key: str, async_session: AsyncSession -) -> Base: + sql_model: Type[T], fides_key: str, async_session: AsyncSession +) -> T: """ Delete a resource by its fides_key. @@ -329,7 +332,7 @@ async def delete_resource( if hasattr(sql_model, "parent_key"): log.debug("Deleting resource and its children") query = ( - _delete(sql_model) + _delete(sql_model.__table__) .where( or_( sql_model.fides_key == fides_key, @@ -340,7 +343,9 @@ async def delete_resource( ) else: log.debug("Deleting resource") - query = _delete(sql_model).where(sql_model.fides_key == fides_key) + query = _delete(sql_model.__table__).where( + sql_model.fides_key == fides_key + ) await async_session.execute(query) except IntegrityError as err: raw_error_text: str = err.orig.args[0] diff --git a/src/fides/api/db/samples.py b/src/fides/api/db/samples.py index 46cc3a12f2..65536462a4 100644 --- a/src/fides/api/db/samples.py +++ b/src/fides/api/db/samples.py @@ -5,16 +5,14 @@ See load_samples() in seed.py for usage. """ -from typing import Dict, List, Optional, TextIO +from importlib.resources import files +from typing import IO, Dict, List, Optional import yaml from expandvars import expandvars # type: ignore from fideslang.models import Taxonomy from fideslang.validation import FidesKey -# DEFER: This can be changed to importlib.resources once we drop support for Python 3.8 -from importlib_resources import files - from fides.api.schemas.connection_configuration.connection_config import ( CreateConnectionConfigurationWithSecrets, ) @@ -104,7 +102,7 @@ def load_sample_connections_from_project() -> List[SampleConnection]: yaml_dict = load_sample_yaml_file(file, expand_vars=True) connections = yaml_dict.get("connection", []) sample_connections.extend( - [SampleConnection.parse_obj(e) for e in connections] + [SampleConnection.model_validate(e) for e in connections] ) # Exclude any connections whose "secrets" dict has empty values @@ -122,7 +120,7 @@ def load_sample_connections_from_project() -> List[SampleConnection]: return valid_connections -def load_sample_yaml_file(file: TextIO, expand_vars: bool = True) -> Dict: +def load_sample_yaml_file(file: IO, expand_vars: bool = True) -> Dict: yaml_str = file.read() if expand_vars: return yaml.safe_load(expandvars(yaml_str)) diff --git a/src/fides/api/db/seed.py b/src/fides/api/db/seed.py index 8d0a877995..d549521016 100644 --- a/src/fides/api/db/seed.py +++ b/src/fides/api/db/seed.py @@ -331,7 +331,8 @@ def load_default_dsr_policies() -> None: "user.authorization", ] all_data_categories = [ - str(category.fides_key) for category in DEFAULT_TAXONOMY.data_category + str(category.fides_key) + for category in DEFAULT_TAXONOMY.data_category # pylint:disable=not-an-iterable ] default_data_categories = filter_data_categories( all_data_categories, excluded_data_categories @@ -353,7 +354,9 @@ async def load_default_organization(async_session: AsyncSession) -> None: """ log.info("Loading the default organization...") - organizations: List[Dict] = list(map(dict, DEFAULT_TAXONOMY.dict()["organization"])) + organizations: List[Dict] = list( + map(dict, DEFAULT_TAXONOMY.model_dump(mode="json")["organization"]) + ) inserted = 0 for org in organizations: @@ -377,13 +380,13 @@ async def load_default_organization(async_session: AsyncSession) -> None: async def load_default_taxonomy(async_session: AsyncSession) -> None: """Seed the database with the default taxonomy resources.""" - upsert_resource_types = list(DEFAULT_TAXONOMY.__fields_set__) + upsert_resource_types = list(DEFAULT_TAXONOMY.model_fields_set) upsert_resource_types.remove("organization") log.info("Loading the default fideslang taxonomy resources...") for resource_type in upsert_resource_types: log.debug(f"Processing {resource_type} resources...") - default_resources = DEFAULT_TAXONOMY.dict()[resource_type] + default_resources = DEFAULT_TAXONOMY.model_dump(mode="json")[resource_type] existing_resources = await list_resource( sql_model_map[resource_type], async_session ) @@ -430,7 +433,7 @@ async def load_samples(async_session: AsyncSession) -> None: else: await upsert_resources( sql_model_map[resource_type], - [e.dict() for e in resources], + [e.model_dump(mode="json") for e in resources], async_session, ) except QueryError: # pragma: no cover @@ -476,7 +479,7 @@ async def load_samples(async_session: AsyncSession) -> None: instantiate_connection_from_template( db=db_session, saas_connector_type=connection.saas_connector_type, - template_values=SaasConnectionTemplateValues.parse_obj( + template_values=SaasConnectionTemplateValues.model_validate( saas_template_data ), ) @@ -505,7 +508,7 @@ async def load_samples(async_session: AsyncSession) -> None: patch_connection_configs( db=db_session, configs=[ - CreateConnectionConfigurationWithSecrets.parse_obj( + CreateConnectionConfigurationWithSecrets.model_validate( connection_config_data ) ], diff --git a/src/fides/api/db/system.py b/src/fides/api/db/system.py index 4d662fdfb6..47166cd645 100644 --- a/src/fides/api/db/system.py +++ b/src/fides/api/db/system.py @@ -154,7 +154,7 @@ async def upsert_privacy_declarations( # looking for "matching" existing declarations based on data_use and name for privacy_declaration in resource.privacy_declarations: # prepare our 'payload' for either create or update - data = privacy_declaration.dict() + data = privacy_declaration.model_dump(mode="json") privacy_declaration_cookies: List[Dict] = data.pop("cookies", None) data["system_id"] = system.id # include FK back to system @@ -203,7 +203,7 @@ async def upsert_cookies( ) parsed_cookies = ( - [CookieSchema.parse_obj(cookie) for cookie in cookies] if cookies else [] + [CookieSchema.model_validate(cookie) for cookie in cookies] if cookies else [] ) resource_filter: BinaryExpression = ( @@ -223,7 +223,9 @@ async def upsert_cookies( if row: # Update existing cookie await async_session.execute( - update(Cookies).where(Cookies.id == row.id).values(cookie_data.dict()) + update(Cookies) + .where(Cookies.id == row.id) + .values(cookie_data.model_dump(mode="json")) ) else: @@ -269,7 +271,9 @@ async def update_system( system: System = await get_resource( sql_model=System, fides_key=resource.fides_key, async_session=db ) - existing_system_dict = copy.deepcopy(SystemSchema.from_orm(system).dict()) + existing_system_dict = copy.deepcopy( + SystemSchema.model_validate(system) + ).model_dump(mode="json") # handle the privacy declaration upsert logic try: @@ -290,7 +294,7 @@ async def update_system( delattr(resource, "cookies") # perform any updates on the system resource itself - updated_system = await update_resource(System, resource.dict(), db) + updated_system = await update_resource(System, resource.model_dump(), db) async with db.begin(): await upsert_cookies( @@ -304,7 +308,7 @@ async def update_system( system.id, current_user_id, existing_system_dict, - SystemSchema.from_orm(updated_system).dict(), + SystemSchema.model_validate(updated_system).model_dump(mode="json"), ) return updated_system, system_updated @@ -402,7 +406,9 @@ async def create_system( # create the system resource using generic creation # the system must be created before the privacy declarations so that it can be referenced - resource_dict = resource.dict() + resource_dict = resource.model_dump( + mode="json" + ) # mode=json helps Url fields be converted to strings before saving to db # set the current user's ID resource_dict["user_id"] = current_user_id @@ -423,7 +429,7 @@ async def create_system( # create the specified declarations as records in their own table for privacy_declaration in privacy_declarations: - data = privacy_declaration.dict() + data = privacy_declaration.model_dump(mode="json") data["system_id"] = created_system.id # add FK back to system privacy_declaration_cookies: List[Dict] = data.pop("cookies", []) privacy_declaration = PrivacyDeclaration.create( diff --git a/src/fides/api/graph/config.py b/src/fides/api/graph/config.py index 991d574c60..71973bb3de 100644 --- a/src/fides/api/graph/config.py +++ b/src/fides/api/graph/config.py @@ -85,7 +85,7 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union from fideslang.validation import FidesKey -from pydantic import BaseModel, validator +from pydantic import BaseModel, ConfigDict, field_serializer, field_validator from fides.api.common_exceptions import FidesopsException from fides.api.graph.data_type import ( @@ -257,24 +257,34 @@ class Field(BaseModel, ABC): """references to other fields in any other datasets""" identity: Optional[SeedAddress] = None """an optional pointer to an arbitrary key in an expected json package provided as a seed value""" - data_categories: Optional[List[FidesKey]] + data_categories: Optional[List[FidesKey]] = None data_type_converter: DataTypeConverter = DataType.no_op.value return_all_elements: Optional[bool] = None # Should field be returned by query if it is in an entrypoint array field, or just if it matches query? """Known type of held data""" - length: Optional[int] + length: Optional[int] = None """Known length of held data""" is_array: bool = False read_only: Optional[bool] = None """Optionally specify if a field is read-only, meaning it can't be updated or deleted. """ - - class Config: - """for pydantic incorporation of custom non-pydantic types""" - - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) + + @field_serializer("data_type_converter") + def serialize_data_type_converter( + self, data_type_converter: DataTypeConverter + ) -> Optional[str]: + """Help Pydantic V2 serialize this unknown type""" + return data_type_converter.name if data_type_converter.name else None + + @field_serializer("references") + def serialize_references( + self, references: List[Tuple[FieldAddress, Optional[EdgeDirection]]] + ) -> List[Tuple[str, Optional[str]]]: + """Help Pydantic V2 serialize this unknown type""" + return [(ref[0].value, ref[1] if ref else None) for ref in references] @abstractmethod def cast(self, value: Any) -> Optional[Any]: @@ -284,6 +294,7 @@ def data_type(self) -> str: """return the data type name""" return self.data_type_converter.name + @abstractmethod def collect_matching(self, func: Callable[[Field], bool]) -> Dict[FieldPath, Field]: """Find fields or subfields satisfying the input function""" @@ -323,7 +334,7 @@ class ObjectField(Field): fields: Dict[str, Field] - @validator("data_categories") + @field_validator("data_categories") @classmethod def validate_data_categories( cls, value: Optional[List[FidesKey]] @@ -550,12 +561,12 @@ def build_field(serialized_field: dict) -> Field: field_name: build_field(fld) for field_name, fld in serialized_field["fields"].items() } - converted = ObjectField.parse_obj(serialized_field) + converted = ObjectField.model_validate(serialized_field) converted.references = converted_references converted.data_type_converter = data_type_converter return converted - converted = ScalarField.parse_obj(serialized_field) + converted = ScalarField.model_validate(serialized_field) converted.references = converted_references converted.data_type_converter = data_type_converter return converted @@ -574,22 +585,38 @@ def build_field(serialized_field: dict) -> Field: for addr_string in data.get("erase_after", []) } - return Collection.parse_obj(data) + return Collection.model_validate(data) - class Config: - """for pydantic incorporation of custom non-pydantic types""" + @field_serializer("data_type_converter", check_fields=False) + def serialize_data_type_converter( + self, data_type_converter: DataTypeConverter + ) -> Optional[str]: + """Help Pydantic V2 serialize this unknown type""" + return data_type_converter.name if data_type_converter.name else None - arbitrary_types_allowed = True + @field_serializer("after") + def serialize_after(self, after: Set[CollectionAddress]) -> Set[str]: + """Help Pydantic V2 serialize this unknown type""" + return {aft.value for aft in after} + + @field_serializer("erase_after") + def serialize_erase_after(self, erase_after: Set[CollectionAddress]) -> Set[str]: + """Help Pydantic V2 serialize this unknown type""" + return {aft.value for aft in erase_after} + + model_config = ConfigDict( + arbitrary_types_allowed=True, # This supports running Collection.json() to serialize less standard # types so it can be saved to the database under RequestTask.collection - json_encoders = { + json_encoders={ Set: lambda val: list( # pylint: disable=unhashable-member,unnecessary-lambda val ), DataTypeConverter: lambda dtc: dtc.name if dtc.name else None, FieldAddress: lambda fa: fa.value, CollectionAddress: lambda ca: ca.value, - } + }, + ) class GraphDataset(BaseModel): diff --git a/src/fides/api/graph/execution.py b/src/fides/api/graph/execution.py index 6d95b6ee15..ae92bfc65d 100644 --- a/src/fides/api/graph/execution.py +++ b/src/fides/api/graph/execution.py @@ -32,7 +32,7 @@ def __init__(self, request_task: RequestTask): self.address: CollectionAddress = CollectionAddress.from_string( request_task.collection_address ) - traversal_details = TraversalDetails.parse_obj( + traversal_details = TraversalDetails.model_validate( request_task.traversal_details or {} ) diff --git a/src/fides/api/graph/traversal.py b/src/fides/api/graph/traversal.py index 73ef2babd1..83cd2c7614 100644 --- a/src/fides/api/graph/traversal.py +++ b/src/fides/api/graph/traversal.py @@ -165,13 +165,17 @@ def format_traversal_details_for_save(self) -> Dict: [edge.f1.value, edge.f2.value] for edge in self.outgoing_edges() ], input_keys=[tn.value for tn in self.input_keys()], - ).dict() + ).model_dump(mode="json") def to_mock_request_task(self) -> RequestTask: """Converts a portion of the TraversalNode into a RequestTask - used in building dry run queries or for supporting Deprecated DSR 2.0. Request Tasks were introduced in DSR 3.0 """ - collection_data = json.loads(self.node.collection.json()) + collection_data = json.loads( + # Serializes with duck-typing behavior, no longer the default in Pydantic v2 + # Needed for serializing nested collection fields + self.node.collection.model_dump_json(serialize_as_any=True) + ) return RequestTask( # Mock a RequestTask object in memory collection_address=self.node.address.value, dataset_name=self.node.address.dataset, @@ -326,18 +330,19 @@ def traverse( # pylint: disable=R0914 node_run_fn(n, environment) # delete all edges between the traversal_node that's just run and any completed nodes for finished_node_address, finished_node in finished_nodes.items(): - completed_edges = Edge.delete_edges( + completed_edges: Set[Edge] = Edge.delete_edges( remaining_edges, finished_node_address, cast(TraversalNode, n).address, # type: ignore[redundant-cast] ) - # append edges that end in this traversal_node - for edge in filter( - lambda _edge: _edge.ends_with_collection( + + def edge_ends_with_collection(_edge: Edge) -> bool: + # append edges that end in this traversal_node + return _edge.ends_with_collection( cast(TraversalNode, n).address # type: ignore[redundant-cast] - ), - completed_edges, - ): + ) + + for edge in filter(edge_ends_with_collection, completed_edges): # note, this will not work for self-reference finished_node.add_child(n, edge) # next edges = take all edges including n that are _not_ in edges_from_completed_nodes diff --git a/src/fides/api/main.py b/src/fides/api/main.py index 8912c5089d..8dcb72e30f 100644 --- a/src/fides/api/main.py +++ b/src/fides/api/main.py @@ -7,15 +7,18 @@ from datetime import datetime, timezone from logging import WARNING from time import perf_counter -from typing import Callable, Optional +from typing import AsyncGenerator, Callable, Optional from urllib.parse import unquote -from fastapi import HTTPException, Request, Response, status -from fastapi.responses import FileResponse, HTMLResponse +from fastapi import FastAPI, HTTPException, Request, Response, status +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse from fideslog.sdk.python.event import AnalyticsEvent from loguru import logger from pyinstrument import Profiler from starlette.background import BackgroundTask +from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY from uvicorn import Config, Server import fides @@ -54,7 +57,60 @@ VERSION = fides.__version__ -app = create_fides_app() + +async def lifespan(wrapped_app: FastAPI) -> AsyncGenerator[None, None]: + """Run all of the required setup steps for the webserver. + + **NOTE**: The order of operations here _is_ deliberate + and must be maintained. + """ + start_time = perf_counter() + logger.info("Starting server setup...") + + if not CONFIG.dev_mode: + sys.tracebacklimit = 0 + + log_startup() + + await run_database_startup(wrapped_app) + + check_redis() + + if not scheduler.running: + scheduler.start() + if not async_scheduler.running: + async_scheduler.start() + + initiate_scheduled_batch_email_send() + initiate_poll_for_exited_privacy_request_tasks() + initiate_scheduled_dsr_data_removal() + + logger.debug("Sending startup analytics events...") + # Avoid circular imports + from fides.api.analytics import in_docker_container, send_analytics_event + + await send_analytics_event( + AnalyticsEvent( + docker=in_docker_container(), + event=Event.server_start.value, + event_created_at=datetime.now(tz=timezone.utc), + ) + ) + + # It's just a random bunch of strings when serialized + if not CONFIG.logging.serialization: + logger.info(FIDES_ASCII_ART) + + warn_root_user_enabled() + + logger.info("Fides startup complete! v{}", VERSION) + startup_time = round(perf_counter() - start_time, 3) + logger.info("Server setup completed in {} seconds", startup_time) + yield # All of this happens before the webserver comes up + + +app = create_fides_app(lifespan=lifespan) # type: ignore + if CONFIG.dev_mode: @@ -259,56 +315,6 @@ def warn_root_user_enabled() -> None: ) -@app.on_event("startup") -async def setup_server() -> None: - """Run all of the required setup steps for the webserver. - - **NOTE**: The order of operations here _is_ deliberate - and must be maintained. - """ - start_time = perf_counter() - logger.info("Starting server setup...") - if not CONFIG.dev_mode: - sys.tracebacklimit = 0 - - log_startup() - - await run_database_startup(app) - - check_redis() - - if not scheduler.running: - scheduler.start() - if not async_scheduler.running: - async_scheduler.start() - - initiate_scheduled_batch_email_send() - initiate_poll_for_exited_privacy_request_tasks() - initiate_scheduled_dsr_data_removal() - - logger.debug("Sending startup analytics events...") - # Avoid circular imports - from fides.api.analytics import in_docker_container, send_analytics_event - - await send_analytics_event( - AnalyticsEvent( - docker=in_docker_container(), - event=Event.server_start.value, - event_created_at=datetime.now(tz=timezone.utc), - ) - ) - - # It's just a random bunch of strings when serialized - if not CONFIG.logging.serialization: - logger.info(FIDES_ASCII_ART) - - warn_root_user_enabled() - - logger.info("Fides startup complete! v{}", VERSION) - startup_time = round(perf_counter() - start_time, 3) - logger.info("Server setup completed in {} seconds", startup_time) - - def start_webserver(port: int = 8080) -> None: """Run the webserver.""" check_required_webserver_config_values(config=CONFIG) @@ -343,3 +349,19 @@ async def action_to_audit_log( except Exception as exc: logger.debug(exc) return await call_next(request) + + +@app.exception_handler(RequestValidationError) +async def request_validation_exception_handler( + request: Request, exc: RequestValidationError +) -> JSONResponse: + """Prevents field input and pydantic error message URL from being returned in the response + For example, if someone is creating a user and the request fails, this prevents the user's + password from being returned in the error message + """ + return JSONResponse( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + content={ + "detail": jsonable_encoder(exc.errors(), exclude={"input", "url", "ctx"}) + }, + ) diff --git a/src/fides/api/middleware.py b/src/fides/api/middleware.py index 857e85ced8..ab112499aa 100644 --- a/src/fides/api/middleware.py +++ b/src/fides/api/middleware.py @@ -45,7 +45,7 @@ async def handle_audit_log_resource(request: Request) -> None: await set_body(request, await request.body()) body = await get_body(request) - fides_keys = await extract_data_from_body(body) + fides_keys: List = await extract_data_from_body(body) audit_log_resource_data["fides_keys"] = fides_keys # write record to server @@ -79,7 +79,7 @@ async def extract_data_from_body(body: bytes) -> List: the request found in the request body. """ - fides_keys = [] + fides_keys: List[str] = [] if body: body = json.loads(body) if isinstance(body, dict): diff --git a/src/fides/api/models/application_config.py b/src/fides/api/models/application_config.py index c17a4e4253..e094c493e1 100644 --- a/src/fides/api/models/application_config.py +++ b/src/fides/api/models/application_config.py @@ -4,7 +4,7 @@ from typing import Any, Dict, Iterable, Optional from loguru import logger -from pydantic.utils import deep_update +from pydantic.v1.utils import deep_update from pydash.objects import get from sqlalchemy import Boolean, CheckConstraint, Column from sqlalchemy.ext.mutable import MutableDict diff --git a/src/fides/api/models/client.py b/src/fides/api/models/client.py index 6a4b264c0b..d4c2cb05f1 100644 --- a/src/fides/api/models/client.py +++ b/src/fides/api/models/client.py @@ -2,7 +2,7 @@ import json from datetime import datetime -from typing import Any +from typing import Any, Optional from sqlalchemy import ARRAY, Column, ForeignKey, String from sqlalchemy.ext.declarative import declared_attr @@ -55,8 +55,8 @@ def create_client_and_secret( client_secret_byte_length: int, *, scopes: list[str] | None = None, - fides_key: str = None, - user_id: str = None, + fides_key: Optional[str] = None, + user_id: Optional[str] = None, encoding: str = "UTF-8", roles: list[str] | None = None, systems: list[str] | None = None, diff --git a/src/fides/api/models/connectionconfig.py b/src/fides/api/models/connectionconfig.py index b625b38385..9dfd0e907b 100644 --- a/src/fides/api/models/connectionconfig.py +++ b/src/fides/api/models/connectionconfig.py @@ -240,7 +240,7 @@ def update_saas_config( } updated_secrets = {**default_secrets, **(self.secrets or {})} self.secrets = updated_secrets - self.saas_config = saas_config.dict() + self.saas_config = saas_config.model_dump(mode="json") self.save(db) def update_test_status( diff --git a/src/fides/api/models/consent_automation.py b/src/fides/api/models/consent_automation.py index 1cab6ce0f1..c308327553 100644 --- a/src/fides/api/models/consent_automation.py +++ b/src/fides/api/models/consent_automation.py @@ -78,7 +78,7 @@ def link_consentable_items_to_consent_automation( } def process_items( - items_data: List[Dict[str, Any]], parent_id: str = None + items_data: List[Dict[str, Any]], parent_id: Optional[str] = None ) -> List[ConsentableItem]: processed_items = [] for item_data in items_data: diff --git a/src/fides/api/models/datasetconfig.py b/src/fides/api/models/datasetconfig.py index 13ae6a0b6a..307d975d0b 100644 --- a/src/fides/api/models/datasetconfig.py +++ b/src/fides/api/models/datasetconfig.py @@ -75,13 +75,13 @@ def upsert_ctl_dataset(ctl_dataset_obj: Optional[CtlDataset]) -> CtlDataset: if ctl_dataset_obj: # It's possible this updates the ctl_dataset.fides_key and this causes a conflict # with another ctl_dataset, if we fetched the datasetconfig.ctl_dataset. - for key, val in validated_data.dict().items(): + for key, val in validated_data.model_dump(mode="json").items(): setattr( ctl_dataset_obj, key, val ) # Just update the existing ctl_dataset with the new values else: ctl_dataset_obj = CtlDataset( - **validated_data.dict() + **validated_data.model_dump(mode="json") ) # Validate the values if creating a new CtlDataset db.add(ctl_dataset_obj) @@ -149,7 +149,7 @@ def get_graph(self) -> GraphDataset: the corresponding SaaS config is merged in as well """ dataset_graph = convert_dataset_to_graph( - Dataset.from_orm(self.ctl_dataset), self.connection_config.key # type: ignore + Dataset.model_validate(self.ctl_dataset), self.connection_config.key # type: ignore ) if ( self.connection_config.connection_type == ConnectionType.saas @@ -230,7 +230,16 @@ def to_graph_field( # becomes: (mongo_example_test_dataset, customer_details, extra.meta.created) (ref_collection, *ref_fields) = reference.field.split(".") address = FieldAddress(reference.dataset, ref_collection, *ref_fields) - references.append((address, reference.direction)) + references.append( + ( + address, + ( + reference.direction.value # Transforming reference back to a literal first for Pydantic v2 + if reference.direction + else reference.direction + ), + ) + ) if meta_section.length is not None: # 'if meta_section.length' will not suffice here, we will want to pass through # length for any valid integer if it has been set in the config, including 0. @@ -347,7 +356,7 @@ def validate_dataset_reference( ) dataset: GraphDataset = convert_dataset_to_graph( - Dataset.from_orm(dataset_config.ctl_dataset), dataset_config.fides_key # type: ignore[arg-type] + Dataset.model_validate(dataset_config.ctl_dataset), dataset_config.fides_key # type: ignore[arg-type] ) collection_name, *field_name = dataset_reference.field.split(".") if not field_name or not collection_name or not field_name[0]: diff --git a/src/fides/api/models/location_regulation_selections.py b/src/fides/api/models/location_regulation_selections.py index 3124fc8075..c39705827d 100644 --- a/src/fides/api/models/location_regulation_selections.py +++ b/src/fides/api/models/location_regulation_selections.py @@ -332,7 +332,7 @@ def load_locations() -> Dict[str, Location]: _locations = yaml.safe_load(file).get("locations", []) location_dict = {} for location in _locations: - location_dict[location["id"]] = Location.parse_obj(location) + location_dict[location["id"]] = Location.model_validate(location) return location_dict @@ -343,7 +343,7 @@ def load_location_groups() -> Dict[str, LocationGroup]: _location_groups = yaml.safe_load(file).get("location_groups", []) location_group_dict = {} for location_group in _location_groups: - location_group_dict[location_group["id"]] = LocationGroup.parse_obj( + location_group_dict[location_group["id"]] = LocationGroup.model_validate( location_group ) return location_group_dict @@ -377,7 +377,7 @@ def load_regulations() -> Dict[str, LocationRegulationBase]: _regulations = yaml.safe_load(file).get("regulations", []) regulation_dict = {} for regulation in _regulations: - regulation_dict[regulation["id"]] = LocationRegulationBase.parse_obj( + regulation_dict[regulation["id"]] = LocationRegulationBase.model_validate( regulation ) return regulation_dict diff --git a/src/fides/api/models/manual_webhook.py b/src/fides/api/models/manual_webhook.py index a38ced8b83..9f17e02667 100644 --- a/src/fides/api/models/manual_webhook.py +++ b/src/fides/api/models/manual_webhook.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional -from pydantic import BaseConfig, create_model +from pydantic import ConfigDict, create_model from sqlalchemy import Column, ForeignKey, String, or_, text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.mutable import MutableList @@ -32,24 +32,29 @@ class AccessManualWebhook(Base): fields = Column(MutableList.as_mutable(JSONB), nullable=False) - @property - def fields_schema(self) -> FidesSchema: - """Build a dynamic Pydantic schema from fields defined on this webhook""" - - class Config: - extra = "forbid" - - field_definitions: Dict[str, Any] = { + def access_field_definitions(self) -> Dict[str, Any]: + """Shared access field definitions for manual webhook schemas""" + return { field["dsr_package_label"]: (Optional[str], None) for field in self.fields or [] } - ManualWebhookValidationModel = create_model( # type: ignore + def erasure_field_definitions(self) -> Dict[str, Any]: + """Shared erasure field definitions for manual webhook schemas""" + return { + field["dsr_package_label"]: (Optional[bool], None) + for field in self.fields or [] + } + + @property + def fields_schema(self) -> FidesSchema: + """Build a dynamic Pydantic schema from fields defined on this webhook""" + + return create_model( # type: ignore __model_name="ManualWebhookValidationModel", - __config__=Config, - **field_definitions, + __config__=ConfigDict(extra="forbid"), + **self.access_field_definitions(), ) - return ManualWebhookValidationModel @property def erasure_fields_schema(self) -> FidesSchema: @@ -58,39 +63,31 @@ def erasure_fields_schema(self) -> FidesSchema: The fields in the schema for erasure input validation are of type bool, vs str for access input validation. """ - - class Config: - extra = "forbid" - - field_definitions: Dict[str, Any] = { - field["dsr_package_label"]: (Optional[bool], None) - for field in self.fields or [] - } - - ManualWebhookValidationModel = create_model( # type: ignore + return create_model( # type: ignore __model_name="ManualWebhookValidationModel", - __config__=Config, - **field_definitions, + model_config=ConfigDict(extra="forbid"), + **self.erasure_field_definitions(), ) - return ManualWebhookValidationModel @property def fields_non_strict_schema(self) -> FidesSchema: """Returns a dynamic Pydantic Schema for webhook fields that can keep the overlap between fields that are saved and fields that are defined here.""" - schema: FidesSchema = self.fields_schema - # Extra is set to "ignore" on the BaseConfig - schema.__config__ = BaseConfig # type: ignore[misc] - return schema + return create_model( # type: ignore + __model_name="ManualWebhookValidationModel", + __config__=ConfigDict(extra="ignore"), + **self.access_field_definitions(), + ) @property def erasure_fields_non_strict_schema(self) -> FidesSchema: """Returns a dynamic Pydantic Schema for webhook fields that can keep the overlap between fields that are saved and fields that are defined here.""" - schema: FidesSchema = self.erasure_fields_schema - # Extra is set to "ignore" on the BaseConfig - schema.__config__ = BaseConfig # type: ignore[misc] - return schema + return create_model( # type: ignore + __model_name="ManualWebhookValidationModel", + model_config=ConfigDict(extra="ignore"), + **self.erasure_field_definitions(), + ) @property def empty_fields_dict(self) -> Dict[str, None]: diff --git a/src/fides/api/models/messaging.py b/src/fides/api/models/messaging.py index 7ba8653789..130b2363aa 100644 --- a/src/fides/api/models/messaging.py +++ b/src/fides/api/models/messaging.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Optional +from typing import Dict, Optional, Type, Union from loguru import logger from pydantic import ValidationError @@ -53,25 +53,26 @@ def get_schema_for_secrets( Returns the secrets that pertain to `service_type` represented as a Pydantic schema for validation purposes. """ + messaging_secret_schema_type = Union[ + Type[MessagingServiceSecretsMailgun], + Type[MessagingServiceSecretsTwilioSMS], + Type[MessagingServiceSecretsTwilioEmail], + Type[MessagingServiceSecretsMailchimpTransactional], + ] try: - schema = { + schema_mapping: Dict[MessagingServiceType, messaging_secret_schema_type] = { MessagingServiceType.mailgun: MessagingServiceSecretsMailgun, MessagingServiceType.twilio_text: MessagingServiceSecretsTwilioSMS, MessagingServiceType.twilio_email: MessagingServiceSecretsTwilioEmail, MessagingServiceType.mailchimp_transactional: MessagingServiceSecretsMailchimpTransactional, - }[service_type] + } + schema: messaging_secret_schema_type = schema_mapping[service_type] except KeyError: raise ValueError( f"`service_type` {service_type} has no supported `secrets` validation." ) - try: - return schema.parse_obj(secrets) # type: ignore - except ValidationError as exc: - # Pydantic requires validators raise either a ValueError, TypeError, or AssertionError - # so this exception is cast into a `ValueError`. - errors = [f"{err['msg']} {str(err['loc'])}" for err in exc.errors()] - raise ValueError(errors) + return schema.model_validate(secrets) class MessagingConfig(Base): diff --git a/src/fides/api/models/messaging_template.py b/src/fides/api/models/messaging_template.py index 00c4e2e120..7abfd96a35 100644 --- a/src/fides/api/models/messaging_template.py +++ b/src/fides/api/models/messaging_template.py @@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional, Type +from pydantic import ConfigDict from sqlalchemy import Boolean, Column, String from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.declarative import declared_attr @@ -94,9 +95,7 @@ def __tablename__(self) -> str: lazy="selectin", ) is_enabled = Column(Boolean, default=False, nullable=False) - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) @classmethod def create( diff --git a/src/fides/api/models/policy.py b/src/fides/api/models/policy.py index 24b0d078f9..3dc643db62 100644 --- a/src/fides/api/models/policy.py +++ b/src/fides/api/models/policy.py @@ -337,6 +337,8 @@ def create_or_update(cls, db: Session, *, data: Dict[str, Any]) -> FidesBase: # `Rule`s to unexpectedly bounce between `Policy`ies. """ db_obj = None + identifier = None + if data.get("id") is not None: # If `id` has been included in `data`, preference that db_obj = cls.get(db=db, object_id=data["id"]) @@ -422,6 +424,8 @@ def create_or_update(cls, db: Session, *, data: Dict[str, Any]) -> FidesBase: # `RuleTarget`s to unexpectedly bounce between `Rule`s. """ db_obj = None + identifier = None + if data.get("id") is not None: # If `id` has been included in `data`, preference that db_obj = cls.get(db=db, object_id=data["id"]) diff --git a/src/fides/api/models/privacy_notice.py b/src/fides/api/models/privacy_notice.py index df934a4ec0..0744fd2a2b 100644 --- a/src/fides/api/models/privacy_notice.py +++ b/src/fides/api/models/privacy_notice.py @@ -4,7 +4,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Set, Type -from fideslang.validation import FidesKey +from fideslang.validation import FidesKey, validate_fides_key from sqlalchemy import Boolean, Column from sqlalchemy import Enum as EnumColumn from sqlalchemy import Float, ForeignKey, String, UniqueConstraint, or_ @@ -111,7 +111,7 @@ def generate_notice_key(cls, name: Optional[str]) -> FidesKey: if not isinstance(name, str): raise Exception("Privacy notice keys must be generated from a string.") notice_key: str = re.sub(r"\s+", "_", name.lower().strip()) - return FidesKey(FidesKey.validate(notice_key)) + return FidesKey(validate_fides_key(notice_key)) class PrivacyNoticeTemplate(PrivacyNoticeBase, Base): @@ -204,7 +204,9 @@ def systems_applicable(self) -> bool: @property def configured_regions(self) -> List[PrivacyNoticeRegion]: """Convenience property to look up which regions are using these Notices.""" - from fides.api.models.privacy_experience import PrivacyExperience + from fides.api.models.privacy_experience import ( # pylint: disable=cyclic-import + PrivacyExperience, + ) db = Session.object_session(self) configured_regions = ( diff --git a/src/fides/api/models/privacy_request.py b/src/fides/api/models/privacy_request.py index e1a878a727..945b1aa666 100644 --- a/src/fides/api/models/privacy_request.py +++ b/src/fides/api/models/privacy_request.py @@ -9,7 +9,7 @@ from celery.result import AsyncResult from loguru import logger -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from sqlalchemy import ( Boolean, Column, @@ -139,11 +139,9 @@ class CheckpointActionRequired(FidesSchema): """ step: CurrentStep - collection: Optional[CollectionAddress] + collection: Optional[CollectionAddress] = None action_needed: Optional[List[ManualAction]] = None - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) EmailRequestFulfillmentBodyParams = Dict[ @@ -189,12 +187,8 @@ class SecondPartyRequestFormat(BaseModel): direction: WebhookDirection callback_type: CallbackType identity: Identity - policy_action: Optional[ActionType] - - class Config: - """Using enum values""" - - use_enum_values = True + policy_action: Optional[ActionType] = None + model_config = ConfigDict(use_enum_values=True) def generate_request_callback_resume_jwe(webhook: PolicyPreWebhook) -> str: @@ -207,7 +201,7 @@ def generate_request_callback_resume_jwe(webhook: PolicyPreWebhook) -> str: iat=datetime.now().isoformat(), ) return generate_jwe( - json.dumps(jwe.dict()), + json.dumps(jwe.model_dump(mode="json")), CONFIG.security.app_encryption_key, ) @@ -222,7 +216,7 @@ def generate_request_callback_pre_approval_jwe(webhook: PreApprovalWebhook) -> s iat=datetime.now().isoformat(), ) return generate_jwe( - json.dumps(jwe.dict()), + json.dumps(jwe.model_dump(mode="json")), CONFIG.security.app_encryption_key, ) @@ -238,7 +232,7 @@ def generate_request_task_callback_jwe(request_task: RequestTask) -> str: iat=datetime.now().isoformat(), ) return generate_jwe( - json.dumps(jwe.dict()), + json.dumps(jwe.model_dump(mode="json")), CONFIG.security.app_encryption_key, ) @@ -428,7 +422,9 @@ def delete(self, db: Session) -> None: provided_identity.delete(db=db) super().delete(db=db) - def cache_identity(self, identity: Identity) -> None: + def cache_identity( + self, identity: Union[Identity, Dict[str, LabeledIdentity]] + ) -> None: """Sets the identity's values at their specific locations in the Fides app cache""" cache: FidesopsRedis = get_cache() @@ -764,11 +760,11 @@ def cache_manual_webhook_access_input( Dynamically creates a Pydantic model from the manual_webhook to use to validate the input_data """ cache: FidesopsRedis = get_cache() - parsed_data = manual_webhook.fields_schema.parse_obj(input_data) + parsed_data = manual_webhook.fields_schema.model_validate(input_data) cache.set_encoded_object( f"WEBHOOK_MANUAL_ACCESS_INPUT__{self.id}__{manual_webhook.id}", - parsed_data.dict(), + parsed_data.model_dump(mode="json"), ) def cache_manual_webhook_erasure_input( @@ -780,11 +776,11 @@ def cache_manual_webhook_erasure_input( Dynamically creates a Pydantic model from the manual_webhook to use to validate the input_data """ cache: FidesopsRedis = get_cache() - parsed_data = manual_webhook.erasure_fields_schema.parse_obj(input_data) + parsed_data = manual_webhook.erasure_fields_schema.model_validate(input_data) cache.set_encoded_object( f"WEBHOOK_MANUAL_ERASURE_INPUT__{self.id}__{manual_webhook.id}", - parsed_data.dict(), + parsed_data.model_dump(mode="json"), ) def get_manual_webhook_access_input_strict( @@ -802,10 +798,12 @@ def get_manual_webhook_access_input_strict( ) if cached_results: - data: Dict[str, Any] = manual_webhook.fields_schema.parse_obj( + data: Dict[str, Any] = manual_webhook.fields_schema.model_validate( cached_results - ).dict(exclude_unset=True) - if set(data.keys()) != set(manual_webhook.fields_schema.__fields__.keys()): + ).model_dump(exclude_unset=True) + if set(data.keys()) != set( + manual_webhook.fields_schema.model_fields.keys() + ): raise ManualWebhookFieldsUnset( f"Fields unset for privacy_request_id '{self.id}' for connection config '{manual_webhook.connection_config.key}'" ) @@ -829,11 +827,11 @@ def get_manual_webhook_erasure_input_strict( ) if cached_results: - data: Dict[str, Any] = manual_webhook.erasure_fields_schema.parse_obj( + data: Dict[str, Any] = manual_webhook.erasure_fields_schema.model_validate( cached_results - ).dict(exclude_unset=True) + ).model_dump(exclude_unset=True) if set(data.keys()) != set( - manual_webhook.erasure_fields_schema.__fields__.keys() + manual_webhook.erasure_fields_schema.model_fields.keys() ): raise ManualWebhookFieldsUnset( f"Fields unset for privacy_request_id '{self.id}' for connection config '{manual_webhook.connection_config.key}'" @@ -855,9 +853,9 @@ def get_manual_webhook_access_input_non_strict( privacy_request=self, manual_webhook=manual_webhook ) if cached_results: - return manual_webhook.fields_non_strict_schema.parse_obj( + return manual_webhook.fields_non_strict_schema.model_validate( cached_results - ).dict() + ).model_dump(mode="json") return manual_webhook.empty_fields_dict def get_manual_webhook_erasure_input_non_strict( @@ -872,9 +870,9 @@ def get_manual_webhook_erasure_input_non_strict( privacy_request=self, manual_webhook=manual_webhook ) if cached_results: - return manual_webhook.erasure_fields_non_strict_schema.parse_obj( + return manual_webhook.erasure_fields_non_strict_schema.model_validate( cached_results - ).dict() + ).model_dump(mode="json") return manual_webhook.empty_fields_dict def cache_data_use_map(self, value: Dict[str, Set[str]]) -> None: @@ -930,7 +928,7 @@ def trigger_pre_approval_webhook( self.id, ) https_connector.execute( # type: ignore - request_body.dict(), + request_body.model_dump(mode="json"), response_expected=False, additional_headers=headers, ) @@ -972,7 +970,7 @@ def trigger_policy_webhook( "Calling webhook '{}' for privacy_request '{}'", webhook.key, self.id ) response: Optional[SecondPartyResponseFormat] = https_connector.execute( # type: ignore - request_body.dict(), + request_body.model_dump(mode="json"), response_expected=response_expected, additional_headers=headers, ) @@ -983,7 +981,7 @@ def trigger_policy_webhook( # Cache any new identities if response_body.derived_identity and any( - [response_body.derived_identity.dict().values()] + [response_body.derived_identity.model_dump(mode="json").values()] ): logger.info( "Updating known identities on privacy request '{}' from webhook '{}'.", @@ -1614,7 +1612,7 @@ def cache_action_required( cache.set_encoded_object( cache_key, - action_required.dict() if action_required else None, + action_required.model_dump() if action_required else None, ) diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index 52dea99c9a..bbc4f9a9eb 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -265,7 +265,7 @@ class Dataset(Base, FidesBase): def create_from_dataset_dict(cls, db: Session, dataset: dict) -> "Dataset": """Add a method to create directly using a synchronous session""" validated_dataset: FideslangDataset = FideslangDataset(**dataset) - ctl_dataset = cls(**validated_dataset.dict()) + ctl_dataset = cls(**validated_dataset.model_dump(mode="json")) db.add(ctl_dataset) db.commit() db.refresh(ctl_dataset) @@ -277,7 +277,7 @@ def field_data_categories(self) -> Set[str]: data_categories = set() for collection in self.collections: dataset_collection = FideslangDatasetCollection(**collection) - for field in dataset_collection.fields: + for field in dataset_collection.fields: # pylint:disable=not-an-iterable if field.data_categories is not None: data_categories.update(field.data_categories) return data_categories @@ -618,14 +618,14 @@ async def get_purpose_legal_basis_override(self) -> Optional[str]: class SystemModel(BaseModel): fides_key: str - meta: Optional[Dict[str, Any]] - fidesctl_meta: Optional[Dict[str, Any]] + meta: Optional[Dict[str, Any]] = None + fidesctl_meta: Optional[Dict[str, Any]] = None system_type: str - privacy_declarations: Optional[Dict[str, Any]] - administrating_department: Optional[str] - egress: Optional[Dict[str, Any]] - ingress: Optional[Dict[str, Any]] - value: Optional[List[Any]] + privacy_declarations: Optional[Dict[str, Any]] = None + administrating_department: Optional[str] = None + egress: Optional[Dict[str, Any]] = None + ingress: Optional[Dict[str, Any]] = None + value: Optional[List[Any]] = None class SystemScans(Base): diff --git a/src/fides/api/oauth/system_manager_oauth_util.py b/src/fides/api/oauth/system_manager_oauth_util.py index 2969bae50b..63912aded6 100644 --- a/src/fides/api/oauth/system_manager_oauth_util.py +++ b/src/fides/api/oauth/system_manager_oauth_util.py @@ -49,7 +49,7 @@ def _get_system_from_request_body( This function is passed as a *dependency* into verify_oauth_client_for_system_from_request_body. """ resp = SystemAuthContainer(original_data=system_data, system=None) - resource_dict = system_data.dict() + resource_dict = system_data.model_dump(mode="json") if resource_dict.get("fides_key"): system = ( db.query(System) diff --git a/src/fides/api/oauth/utils.py b/src/fides/api/oauth/utils.py index 6803635219..a45989b21c 100644 --- a/src/fides/api/oauth/utils.py +++ b/src/fides/api/oauth/utils.py @@ -66,9 +66,9 @@ def copy_func(source_function: Callable) -> Callable: argdefs=source_function.__defaults__, closure=source_function.__closure__, ) - target_function = update_wrapper(target_function, source_function) - target_function.__kwdefaults__ = source_function.__kwdefaults__ - return target_function + updated_target_function: Callable = update_wrapper(target_function, source_function) + updated_target_function.__kwdefaults__ = source_function.__kwdefaults__ + return updated_target_function async def get_current_user( diff --git a/src/fides/api/schemas/api.py b/src/fides/api/schemas/api.py index d669bc270f..4455b832ba 100644 --- a/src/fides/api/schemas/api.py +++ b/src/fides/api/schemas/api.py @@ -15,9 +15,10 @@ class BulkResponse(BaseModel): failed: List[BulkUpdateFailed] """ - def __init_subclass__(cls: BaseModel, **kwargs: Any): # type: ignore - super().__init_subclass__(**kwargs) # type: ignore - if "succeeded" not in cls.__fields__ or "failed" not in cls.__fields__: + @classmethod + def __pydantic_init_subclass__(cls: BaseModel, **kwargs): # type: ignore + super().__pydantic_init_subclass__(**kwargs) # type: ignore + if "succeeded" not in cls.model_fields or "failed" not in cls.model_fields: raise TypeError( f"Class {cls.__name__} needs both 'succeeded' and 'failed' attributes defined." # type: ignore ) diff --git a/src/fides/api/schemas/application_config.py b/src/fides/api/schemas/application_config.py index 5f00ef7cc4..e56b0cb9ec 100644 --- a/src/fides/api/schemas/application_config.py +++ b/src/fides/api/schemas/application_config.py @@ -3,9 +3,9 @@ from enum import Enum from typing import Dict, List, Optional -from pydantic import Extra, Field, root_validator, validator +from pydantic import ConfigDict, Field, SerializeAsAny, field_validator, model_validator -from fides.api.custom_types import URLOrigin +from fides.api.custom_types import AnyHttpUrlStringRemovesSlash, URLOriginString from fides.api.schemas.base_class import FidesSchema from fides.api.schemas.messaging.messaging import MessagingServiceType @@ -19,10 +19,7 @@ class StorageTypeApiAccepted(Enum): class StorageApplicationConfig(FidesSchema): active_default_storage_type: StorageTypeApiAccepted - - class Config: - use_enum_values = True - extra = Extra.forbid + model_config = ConfigDict(use_enum_values=True, extra="forbid") # TODO: the below models classes are "duplicates" of the pydantic @@ -37,16 +34,14 @@ class NotificationApplicationConfig(FidesSchema): API model - configuration settings for data subject and/or data processor notifications """ - send_request_completion_notification: Optional[bool] - send_request_receipt_notification: Optional[bool] - send_request_review_notification: Optional[bool] - notification_service_type: Optional[str] - enable_property_specific_messaging: Optional[bool] - - class Config: - extra = Extra.forbid + send_request_completion_notification: Optional[bool] = None + send_request_receipt_notification: Optional[bool] = None + send_request_review_notification: Optional[bool] = None + notification_service_type: Optional[str] = None + enable_property_specific_messaging: Optional[bool] = None + model_config = ConfigDict(extra="forbid") - @validator("notification_service_type", pre=True) + @field_validator("notification_service_type", mode="before") @classmethod def validate_notification_service_type(cls, value: str) -> Optional[str]: """Ensure the provided type is a valid value.""" @@ -62,27 +57,22 @@ def validate_notification_service_type(cls, value: str) -> Optional[str]: class ExecutionApplicationConfig(FidesSchema): - subject_identity_verification_required: Optional[bool] - disable_consent_identity_verification: Optional[bool] - require_manual_request_approval: Optional[bool] - - class Config: - extra = Extra.forbid + subject_identity_verification_required: Optional[bool] = None + disable_consent_identity_verification: Optional[bool] = None + require_manual_request_approval: Optional[bool] = None + model_config = ConfigDict(extra="forbid") class AdminUIConfig(FidesSchema): - enabled: Optional[bool] - url: Optional[str] + enabled: Optional[bool] = None + url: SerializeAsAny[Optional[AnyHttpUrlStringRemovesSlash]] = None - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class ConsentConfig(FidesSchema): override_vendor_purposes: Optional[bool] - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class SecurityApplicationConfig(FidesSchema): @@ -90,13 +80,11 @@ class SecurityApplicationConfig(FidesSchema): # for advanced usage of non-URLs, e.g. wildcards (`*`), the related # `cors_origin_regex` property should be used. # this is explicitly _not_ accessible via API - it must be used with care. - cors_origins: Optional[List[URLOrigin]] = Field( + cors_origins: SerializeAsAny[Optional[List[URLOriginString]]] = Field( default=None, description="A list of client addresses allowed to communicate with the Fides webserver.", ) - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class ApplicationConfig(FidesSchema): @@ -108,14 +96,15 @@ class ApplicationConfig(FidesSchema): the application config that is properly hooked up to the global pydantic config module. """ - storage: Optional[StorageApplicationConfig] - notifications: Optional[NotificationApplicationConfig] - execution: Optional[ExecutionApplicationConfig] - security: Optional[SecurityApplicationConfig] - consent: Optional[ConsentConfig] - admin_ui: Optional[AdminUIConfig] + storage: Optional[StorageApplicationConfig] = None + notifications: Optional[NotificationApplicationConfig] = None + execution: Optional[ExecutionApplicationConfig] = None + security: Optional[SecurityApplicationConfig] = None + consent: Optional[ConsentConfig] = None + admin_ui: Optional[AdminUIConfig] = None - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def validate_not_empty(cls, values: Dict) -> Dict: if not values: raise ValueError( @@ -123,5 +112,4 @@ def validate_not_empty(cls, values: Dict) -> Dict: ) return values - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") diff --git a/src/fides/api/schemas/base_class.py b/src/fides/api/schemas/base_class.py index bbfed2252d..371546bac7 100644 --- a/src/fides/api/schemas/base_class.py +++ b/src/fides/api/schemas/base_class.py @@ -1,6 +1,7 @@ -from typing import Any, List +from typing import Any, Callable, List -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict +from pydantic_core import core_schema class NoValidationSchema(BaseModel): @@ -11,9 +12,20 @@ class NoValidationSchema(BaseModel): """ @classmethod - def validate(cls: "NoValidationSchema", value: Any) -> Any: # type: ignore - """Returns value exactly as it was passed in, when validation is going to be handled later.""" - return value + def __get_pydantic_core_schema__( # pylint: disable=arguments-differ + cls, source_type: Any, handler: Callable[[Any], core_schema.CoreSchema] + ) -> core_schema.CoreSchema: + """Custom override for Pydantic V2 - allows us to defer validation for + connection secrets schemas until later.""" + + schema = handler(source_type) + + def val(v: Any, _: core_schema.ValidatorFunctionWrapHandler) -> Any: + return v + + return core_schema.no_info_wrap_validator_function( + function=val, schema=schema, serialization=schema.get("serialization") + ) class FidesSchema(BaseModel): @@ -24,7 +36,4 @@ def get_field_names(cls) -> List[str]: """Return a list of all field names specified on this schema.""" return list(cls.schema().get("properties", {}).keys()) - class Config: - """Allow ORM access on all schemas.""" - - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/src/fides/api/schemas/connection_configuration/__init__.py b/src/fides/api/schemas/connection_configuration/__init__.py index 7971b2102e..50e482fafe 100644 --- a/src/fides/api/schemas/connection_configuration/__init__.py +++ b/src/fides/api/schemas/connection_configuration/__init__.py @@ -181,6 +181,10 @@ def get_connection_secrets_schema( ) +# These schemas are used to generate the documentation for the connection secrets +# but validation is intentionally postponed due to the NoValidationSchema. +# Creating the actual connection secrets schemas happens later once we know +# what type of schema we should validate against. connection_secrets_schemas = Union[ MongoDBDocsSchema, PostgreSQLDocsSchema, diff --git a/src/fides/api/schemas/connection_configuration/connection_config.py b/src/fides/api/schemas/connection_configuration/connection_config.py index b831ccf9ac..15ac22f330 100644 --- a/src/fides/api/schemas/connection_configuration/connection_config.py +++ b/src/fides/api/schemas/connection_configuration/connection_config.py @@ -4,7 +4,7 @@ from fideslang.models import Dataset from fideslang.validation import FidesKey from loguru import logger -from pydantic import BaseModel, Extra, root_validator +from pydantic import BaseModel, ConfigDict, model_validator from fides.api.common_exceptions import NoSuchConnectionTypeSecretSchemaError from fides.api.models.connectionconfig import AccessLevel, ConnectionType @@ -22,30 +22,23 @@ class CreateConnectionConfiguration(BaseModel): Note that secrets are *NOT* allowed to be supplied here. """ - name: Optional[str] - key: Optional[FidesKey] + name: Optional[str] = None + key: Optional[FidesKey] = None connection_type: ConnectionType access: AccessLevel disabled: Optional[bool] = False - description: Optional[str] - - class Config: - """Restrict adding other fields through this schema and set orm_mode to support mapping to ConnectionConfig""" - - orm_mode = True - use_enum_values = True - extra = Extra.ignore + description: Optional[str] = None + model_config = ConfigDict( + from_attributes=True, use_enum_values=True, extra="ignore" + ) class CreateConnectionConfigurationWithSecrets(CreateConnectionConfiguration): """Schema for creating a connection configuration including secrets.""" secrets: Optional[connection_secrets_schemas] = None - saas_connector_type: Optional[str] - - class Config: - orm_mode = True - extra = Extra.ignore + saas_connector_type: Optional[str] = None + model_config = ConfigDict(from_attributes=True, extra="ignore") def mask_sensitive_fields( @@ -83,31 +76,31 @@ class ConnectionConfigurationResponse(BaseModel): Describes the returned schema for a ConnectionConfiguration. """ - name: Optional[str] + name: Optional[str] = None key: FidesKey - description: Optional[str] + description: Optional[str] = None connection_type: ConnectionType access: AccessLevel created_at: datetime - updated_at: Optional[datetime] + updated_at: Optional[datetime] = None disabled: Optional[bool] = False - last_test_timestamp: Optional[datetime] - last_test_succeeded: Optional[bool] - saas_config: Optional[SaaSConfigBase] - secrets: Optional[Dict[str, Any]] + last_test_timestamp: Optional[datetime] = None + last_test_succeeded: Optional[bool] = None + saas_config: Optional[SaaSConfigBase] = None + secrets: Optional[Dict[str, Any]] = None authorized: Optional[bool] = False - enabled_actions: Optional[List[ActionType]] + enabled_actions: Optional[List[ActionType]] = None - @root_validator() - def mask_sensitive_values(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @model_validator(mode="after") + def mask_sensitive_values(self) -> "ConnectionConfigurationResponse": """Mask sensitive values in the response.""" - if values.get("secrets") is None: - return values + if self.secrets is None: + return self connection_type = ( - values["saas_config"].type - if values.get("connection_type") == ConnectionType.saas - else values.get("connection_type").value # type: ignore + self.saas_config.type + if self.connection_type == ConnectionType.saas and self.saas_config + else self.connection_type.value # type: ignore ) try: secret_schema = get_connection_type_secret_schema( @@ -117,18 +110,13 @@ def mask_sensitive_values(cls, values: Dict[str, Any]) -> Dict[str, Any]: logger.error(e) # if there is no schema, we don't know what values to mask. # so all the secrets are removed. - values["secrets"] = None - return values - - values["secrets"] = mask_sensitive_fields( - cast(dict, values.get("secrets")), secret_schema - ) - return values + self.secrets = None + return self - class Config: - """Set orm_mode to support mapping to ConnectionConfig""" + self.secrets = mask_sensitive_fields(cast(dict, self.secrets), secret_schema) + return self - orm_mode = True + model_config = ConfigDict(from_attributes=True) class BulkPutConnectionConfiguration(BulkResponse): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets.py b/src/fides/api/schemas/connection_configuration/connection_secrets.py index 0c8bba98d9..515b3271dd 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets.py @@ -1,9 +1,9 @@ from __future__ import annotations import abc -from typing import Any, Dict, List, Optional +from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, Extra, root_validator +from pydantic import BaseModel, ConfigDict, model_validator from fides.api.models.connectionconfig import ConnectionTestStatus from fides.api.schemas import Msg @@ -14,19 +14,19 @@ class ConnectionConfigSecretsSchema(BaseModel, abc.ABC): # NOTE: any fields not listed in `_required_components` must also have an # annotated Optional type, in order to be treated effectively as optional fields - _required_components: List[str] + # Further, annotating this as as a ClassVar prevents this from being converted to a ModelPrivateAttrs + _required_components: ClassVar[List[str]] def __init_subclass__(cls: BaseModel, **kwargs: Any): # type: ignore super().__init_subclass__(**kwargs) # type: ignore if not getattr(cls, "_required_components"): raise TypeError(f"Class {cls.__name__} must define '_required_components.'") # type: ignore - @root_validator + @model_validator(mode="before") @classmethod - def required_components_supplied( # type: ignore - cls: ConnectionConfigSecretsSchema, values: Dict[str, Any] - ) -> Dict[str, Any]: + def required_components_supplied(cls, values) -> Dict[str, Any]: # type: ignore """Validate that the minimum required components have been supplied.""" + min_fields_present = all( values.get(component) for component in cls._required_components ) @@ -37,11 +37,7 @@ def required_components_supplied( # type: ignore return values - class Config: - """Only permit selected secret fields to be stored.""" - - extra = Extra.ignore - orm_mode = True + model_config = ConfigDict(extra="ignore", from_attributes=True) class TestStatusMessage(Msg): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_attentive.py b/src/fides/api/schemas/connection_configuration/connection_secrets_attentive.py index 6e1ddd37c3..c585a3bf19 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_attentive.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_attentive.py @@ -8,7 +8,7 @@ class AttentiveSchema(EmailSchema): third_party_vendor_name: str = "Attentive" - recipient_email_address: EmailStr = EmailStr("privacy@attentive.com") + recipient_email_address: EmailStr = "privacy@attentive.com" class AttentiveDocsSchema(AttentiveSchema, NoValidationSchema): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_bigquery.py b/src/fides/api/schemas/connection_configuration/connection_secrets_bigquery.py index 38c03a74e7..08ad8dd6af 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_bigquery.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_bigquery.py @@ -1,8 +1,8 @@ import json -from typing import List, Optional, Union +from typing import ClassVar, List, Optional, Union from google.cloud.bigquery import Client as BigQueryClient -from pydantic import EmailStr, Field, parse_obj_as, validator +from pydantic import EmailStr, Field, field_validator from pydantic.main import BaseModel from fides.api.schemas.base_class import NoValidationSchema @@ -16,40 +16,49 @@ class KeyfileCreds(BaseModel): type: Optional[str] = None project_id: str = Field(title="Project ID") - private_key_id: Optional[str] = Field(None, title="Private Key ID") - private_key: Optional[str] = Field(None, sensitive=True) + private_key_id: Optional[str] = Field(default=None, title="Private Key ID") + private_key: Optional[str] = Field( + default=None, json_schema_extra={"sensitive": True} + ) client_email: Optional[EmailStr] = None - client_id: Optional[str] = Field(None, title="Client ID") - auth_uri: Optional[str] = Field(None, title="Auth URI") - token_uri: Optional[str] = Field(None, title="Token URI") + client_id: Optional[str] = Field(default=None, title="Client ID") + auth_uri: Optional[str] = Field(default=None, title="Auth URI") + token_uri: Optional[str] = Field(default=None, title="Token URI") auth_provider_x509_cert_url: Optional[str] = Field( - None, title="Auth Provider X509 Cert URL" + default=None, title="Auth Provider X509 Cert URL" + ) + client_x509_cert_url: Optional[str] = Field( + default=None, title="Client X509 Cert URL" ) - client_x509_cert_url: Optional[str] = Field(None, title="Client X509 Cert URL") class BigQuerySchema(ConnectionConfigSecretsSchema): """Schema to validate the secrets needed to connect to BigQuery""" keyfile_creds: KeyfileCreds = Field( - sensitive=True, + title="Keyfile Creds", + json_schema_extra={"sensitive": True}, description="The contents of the key file that contains authentication credentials for a service account in GCP.", ) dataset: Optional[str] = Field( + default=None, title="BigQuery Dataset", description="The dataset within your BigQuery project that contains the tables you want to access.", ) - _required_components: List[str] = ["keyfile_creds"] + _required_components: ClassVar[List[str]] = ["keyfile_creds"] - @validator("keyfile_creds", pre=True) + @field_validator("keyfile_creds", mode="before") + @classmethod def parse_keyfile_creds(cls, v: Union[str, dict]) -> KeyfileCreds: if isinstance(v, str): v = json.loads(v) - return parse_obj_as(KeyfileCreds, v) + return KeyfileCreds.model_validate(v) def get_client(self) -> BigQueryClient: - return BigQueryClient.from_service_account_info(self.keyfile_creds.dict()) + return BigQueryClient.from_service_account_info( + self.keyfile_creds.model_dump() # pylint: disable=no-member + ) class BigQueryDocsSchema(BigQuerySchema, NoValidationSchema): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_dynamodb.py b/src/fides/api/schemas/connection_configuration/connection_secrets_dynamodb.py index d4ba4d626d..1208bc42ab 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_dynamodb.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_dynamodb.py @@ -1,4 +1,4 @@ -from typing import List +from typing import ClassVar, List from pydantic import Field @@ -22,12 +22,12 @@ class DynamoDBSchema(ConnectionConfigSecretsSchema): aws_secret_access_key: str = Field( title="Secret Access Key", description="Part of the credentials that provide access to your AWS account.", - sensitive=True, + json_schema_extra={"sensitive": True}, ) # TODO: include an aws_assume_role_arn and more closely follow the pattern in `connection_secrets_s3` - _required_components: List[str] = [ + _required_components: ClassVar[List[str]] = [ "region_name", "aws_access_key_id", "aws_secret_access_key", diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_email.py b/src/fides/api/schemas/connection_configuration/connection_secrets_email.py index d02b60c08d..489b7a4c94 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_email.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_email.py @@ -1,6 +1,6 @@ -from typing import Any, Dict, List, Optional +from typing import List, Optional -from pydantic import BaseModel, EmailStr, Extra, root_validator +from pydantic import BaseModel, ConfigDict, EmailStr, model_validator from fides.api.schemas.base_class import NoValidationSchema @@ -19,30 +19,27 @@ class EmailSchema(BaseModel): third_party_vendor_name: str recipient_email_address: EmailStr - test_email_address: Optional[EmailStr] # Email to send a connection test email + test_email_address: Optional[EmailStr] = ( + None # Email to send a connection test email + ) # the default value is temporary until we allow users to customize the identity types from the front-end advanced_settings: AdvancedSettings = AdvancedSettings( identity_types=IdentityTypes(email=True, phone_number=False) ) + model_config = ConfigDict(extra="forbid", from_attributes=True) - class Config: - """Only permit selected secret fields to be stored.""" - - extra = Extra.forbid - orm_mode = True - - @root_validator - def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @model_validator(mode="after") + def validate_fields(self) -> "EmailSchema": """At least one identity or browser identity needs to be specified on setup""" - advanced_settings = values.get("advanced_settings") + advanced_settings = self.advanced_settings if not advanced_settings: raise ValueError("Must supply advanced settings.") identities = advanced_settings.identity_types if not identities.email and not identities.phone_number: raise ValueError("Must supply at least one identity_type.") - return values + return self class EmailDocsSchema(EmailSchema, NoValidationSchema): @@ -73,10 +70,10 @@ class ExtendedEmailSchema(EmailSchema): ) ) - @root_validator - def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @model_validator(mode="after") + def validate_fields(self) -> "ExtendedEmailSchema": """At least one identity or browser identity needs to be specified on setup""" - advanced_settings = values.get("advanced_settings") + advanced_settings = self.advanced_settings if not advanced_settings: raise ValueError("Must supply advanced settings.") @@ -87,4 +84,4 @@ def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: and not identities.cookie_ids ): raise ValueError("Must supply at least one identity_type.") - return values + return self diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_fides.py b/src/fides/api/schemas/connection_configuration/connection_secrets_fides.py index 0500f80500..8c8ba6e668 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_fides.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_fides.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import ClassVar, List, Optional from pydantic import Field @@ -13,11 +13,11 @@ class FidesConnectorSchema(ConnectionConfigSecretsSchema): uri: str username: str - password: str = Field(sensitive=True) + password: str = Field(json_schema_extra={"sensitive": True}) polling_timeout: Optional[int] = None polling_interval: Optional[int] = None - _required_components: List[str] = ["uri", "username", "password"] + _required_components: ClassVar[List[str]] = ["uri", "username", "password"] class FidesDocsSchema(FidesConnectorSchema, NoValidationSchema): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_google_cloud_sql_mysql.py b/src/fides/api/schemas/connection_configuration/connection_secrets_google_cloud_sql_mysql.py index c78b0f5d87..2ce3feb2ab 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_google_cloud_sql_mysql.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_google_cloud_sql_mysql.py @@ -1,7 +1,7 @@ import json -from typing import List, Optional, Union +from typing import ClassVar, List, Optional, Union -from pydantic import EmailStr, Field, parse_obj_as, validator +from pydantic import EmailStr, Field, field_validator from pydantic.main import BaseModel from fides.api.schemas.base_class import NoValidationSchema @@ -15,16 +15,20 @@ class KeyfileCreds(BaseModel): type: Optional[str] = None project_id: str = Field(title="Project ID") - private_key_id: Optional[str] = Field(None, title="Private key ID") - private_key: Optional[str] = Field(None, sensitive=True) + private_key_id: Optional[str] = Field(default=None, title="Private key ID") + private_key: Optional[str] = Field( + default=None, json_schema_extra={"sensitive": True} + ) client_email: Optional[EmailStr] = None - client_id: Optional[str] = Field(None, title="Client ID") - auth_uri: Optional[str] = Field(None, title="Auth URI") - token_uri: Optional[str] = Field(None, title="Token URI") + client_id: Optional[str] = Field(default=None, title="Client ID") + auth_uri: Optional[str] = Field(default=None, title="Auth URI") + token_uri: Optional[str] = Field(default=None, title="Token URI") auth_provider_x509_cert_url: Optional[str] = Field( None, title="Auth provider X509 cert URL" ) - client_x509_cert_url: Optional[str] = Field(None, title="Client X509 cert URL") + client_x509_cert_url: Optional[str] = Field( + default=None, title="Client X509 cert URL" + ) universe_domain: str = Field(title="Universe domain") @@ -44,22 +48,23 @@ class GoogleCloudSQLMySQLSchema(ConnectionConfigSecretsSchema): ) keyfile_creds: KeyfileCreds = Field( title="Keyfile creds", - sensitive=True, + json_schema_extra={"sensitive": True}, description="The contents of the key file that contains authentication credentials for a service account in GCP.", ) - _required_components: List[str] = [ + _required_components: ClassVar[List[str]] = [ "db_iam_user", "instance_connection_name", "dbname", "keyfile_creds", ] - @validator("keyfile_creds", pre=True) + @field_validator("keyfile_creds", mode="before") + @classmethod def parse_keyfile_creds(cls, v: Union[str, dict]) -> KeyfileCreds: if isinstance(v, str): v = json.loads(v) - return parse_obj_as(KeyfileCreds, v) + return KeyfileCreds.model_validate(v) class GoogleCloudSQLMySQLDocsSchema(GoogleCloudSQLMySQLSchema, NoValidationSchema): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_google_cloud_sql_postgres.py b/src/fides/api/schemas/connection_configuration/connection_secrets_google_cloud_sql_postgres.py index 5dc80ed725..f7861cb7e1 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_google_cloud_sql_postgres.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_google_cloud_sql_postgres.py @@ -1,7 +1,7 @@ import json -from typing import List, Optional, Union +from typing import ClassVar, List, Optional, Union -from pydantic import EmailStr, Field, parse_obj_as, validator +from pydantic import EmailStr, Field, field_validator from pydantic.main import BaseModel from fides.api.schemas.base_class import NoValidationSchema @@ -15,16 +15,20 @@ class KeyfileCreds(BaseModel): type: Optional[str] = None project_id: str = Field(title="Project ID") - private_key_id: Optional[str] = Field(None, title="Private key ID") - private_key: Optional[str] = Field(None, sensitive=True) + private_key_id: Optional[str] = Field(default=None, title="Private key ID") + private_key: Optional[str] = Field( + default=None, json_schema_extra={"sensitive": True} + ) client_email: Optional[EmailStr] = None - client_id: Optional[str] = Field(None, title="Client ID") - auth_uri: Optional[str] = Field(None, title="Auth URI") - token_uri: Optional[str] = Field(None, title="Token URI") + client_id: Optional[str] = Field(default=None, title="Client ID") + auth_uri: Optional[str] = Field(default=None, title="Auth URI") + token_uri: Optional[str] = Field(default=None, title="Token URI") auth_provider_x509_cert_url: Optional[str] = Field( None, title="Auth provider X509 cert URL" ) - client_x509_cert_url: Optional[str] = Field(None, title="Client X509 cert URL") + client_x509_cert_url: Optional[str] = Field( + default=None, title="Client X509 cert URL" + ) universe_domain: str = Field(title="Universe domain") @@ -43,27 +47,29 @@ class GoogleCloudSQLPostgresSchema(ConnectionConfigSecretsSchema): title="Database name", ) db_schema: Optional[str] = Field( + default=None, title="Schema", description="The default schema to be used for the database connection (defaults to public).", ) keyfile_creds: KeyfileCreds = Field( title="Keyfile creds", - sensitive=True, + json_schema_extra={"sensitive": True}, description="The contents of the key file that contains authentication credentials for a service account in GCP.", ) - _required_components: List[str] = [ + _required_components: ClassVar[List[str]] = [ "db_iam_user", "instance_connection_name", "dbname", "keyfile_creds", ] - @validator("keyfile_creds", pre=True) + @field_validator("keyfile_creds", mode="before") + @classmethod def parse_keyfile_creds(cls, v: Union[str, dict]) -> KeyfileCreds: if isinstance(v, str): v = json.loads(v) - return parse_obj_as(KeyfileCreds, v) + return KeyfileCreds.model_validate(v) class GoogleCloudSQLPostgresDocsSchema( diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_mariadb.py b/src/fides/api/schemas/connection_configuration/connection_secrets_mariadb.py index f2b90ad581..4abc72db16 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_mariadb.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_mariadb.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import ClassVar, List, Optional from pydantic import Field @@ -29,14 +29,14 @@ class MariaDBSchema(ConnectionConfigSecretsSchema): None, title="Password", description="The password used to authenticate and access the database.", - sensitive=True, + json_schema_extra={"sensitive": True}, ) dbname: str = Field( description="The name of the specific database within the database server that you want to connect to.", title="Database", ) - _required_components: List[str] = ["host", "dbname"] + _required_components: ClassVar[List[str]] = ["host", "dbname"] class MariaDBDocsSchema(MariaDBSchema, NoValidationSchema): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_mongodb.py b/src/fides/api/schemas/connection_configuration/connection_secrets_mongodb.py index 4c992d4c1b..1ebd42d5df 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_mongodb.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_mongodb.py @@ -1,4 +1,4 @@ -from typing import List +from typing import ClassVar, List from pydantic import Field @@ -27,14 +27,14 @@ class MongoDBSchema(ConnectionConfigSecretsSchema): password: str = Field( title="Password", description="The password used to authenticate and access the database.", - sensitive=True, + json_schema_extra={"sensitive": True}, ) defaultauthdb: str = Field( title="Default Auth DB", description="Used to specify the default authentication database.", ) - _required_components: List[str] = [ + _required_components: ClassVar[List[str]] = [ "host", "username", "password", diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_mssql.py b/src/fides/api/schemas/connection_configuration/connection_secrets_mssql.py index 1deeec2593..cd51d1f8a1 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_mssql.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_mssql.py @@ -1,4 +1,4 @@ -from typing import List +from typing import ClassVar, List from pydantic import Field @@ -32,14 +32,19 @@ class MicrosoftSQLServerSchema(ConnectionConfigSecretsSchema): password: str = Field( title="Password", description="The password used to authenticate and access the database.", - sensitive=True, + json_schema_extra={"sensitive": True}, ) dbname: str = Field( description="The name of the specific database within the database server that you want to connect to.", title="Database", ) - _required_components: List[str] = ["host", "username", "password", "dbname"] + _required_components: ClassVar[List[str]] = [ + "host", + "username", + "password", + "dbname", + ] class MSSQLDocsSchema(MicrosoftSQLServerSchema, NoValidationSchema): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_mysql.py b/src/fides/api/schemas/connection_configuration/connection_secrets_mysql.py index 44a0c8c010..238949d607 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_mysql.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_mysql.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import ClassVar, List, Optional from pydantic import Field @@ -29,7 +29,7 @@ class MySQLSchema(ConnectionConfigSecretsSchema): None, title="Password", description="The password used to authenticate and access the database.", - sensitive=True, + json_schema_extra={"sensitive": True}, ) dbname: str = Field( description="The name of the specific database within the database server that you want to connect to.", @@ -41,7 +41,7 @@ class MySQLSchema(ConnectionConfigSecretsSchema): description="Indicates whether an SSH tunnel is required for the connection. Enable this option if your MySQL server is behind a firewall and requires SSH tunneling for remote connections.", ) - _required_components: List[str] = ["host", "dbname"] + _required_components: ClassVar[List[str]] = ["host", "dbname"] class MySQLDocsSchema(MySQLSchema, NoValidationSchema): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_postgres.py b/src/fides/api/schemas/connection_configuration/connection_secrets_postgres.py index 0966c79981..af685d1a79 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_postgres.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_postgres.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import ClassVar, List, Optional from pydantic import Field @@ -29,13 +29,14 @@ class PostgreSQLSchema(ConnectionConfigSecretsSchema): None, title="Password", description="The password used to authenticate and access the database.", - sensitive=True, + json_schema_extra={"sensitive": True}, ) dbname: str = Field( title="Database", description="The name of the specific database within the database server that you want to connect to.", ) db_schema: Optional[str] = Field( + default=None, title="Schema", description="The default schema to be used for the database connection (defaults to public).", ) @@ -45,7 +46,7 @@ class PostgreSQLSchema(ConnectionConfigSecretsSchema): description="Indicates whether an SSH tunnel is required for the connection. Enable this option if your PostgreSQL server is behind a firewall and requires SSH tunneling for remote connections.", ) - _required_components: List[str] = [ + _required_components: ClassVar[List[str]] = [ "host", "dbname", ] diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_redshift.py b/src/fides/api/schemas/connection_configuration/connection_secrets_redshift.py index ccff0d3941..99285d5215 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_redshift.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_redshift.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import ClassVar, List, Optional from pydantic import Field @@ -27,7 +27,7 @@ class RedshiftSchema(ConnectionConfigSecretsSchema): password: str = Field( title="Password", description="The password used to authenticate and access the database.", - sensitive=True, + json_schema_extra={"sensitive": True}, ) database: str = Field( title="Database", @@ -44,7 +44,7 @@ class RedshiftSchema(ConnectionConfigSecretsSchema): description="Indicates whether an SSH tunnel is required for the connection. Enable this option if your Redshift database is behind a firewall and requires SSH tunneling for remote connections.", ) - _required_components: List[str] = [ + _required_components: ClassVar[List[str]] = [ "host", "user", "password", diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_s3.py b/src/fides/api/schemas/connection_configuration/connection_secrets_s3.py index 68e49933ae..60fed36c5e 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_s3.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_s3.py @@ -1,6 +1,6 @@ -from typing import Dict, List, Optional +from typing import ClassVar, List, Optional -from pydantic import Field, root_validator +from pydantic import Field, model_validator from fides.api.schemas.base_class import NoValidationSchema from fides.api.schemas.connection_configuration.connection_secrets import ( @@ -18,36 +18,38 @@ class S3Schema(ConnectionConfigSecretsSchema): ) aws_access_key_id: Optional[str] = Field( + default=None, title="Access Key ID", description="Part of the credentials that provide access to your AWS account. This is required if using secret key authentication.", ) aws_secret_access_key: Optional[str] = Field( + default=None, title="Secret Access Key", description="Part of the credentials that provide access to your AWS account. This is required if using secret key authentication.", - sensitive=True, + json_schema_extra={"sensitive": True}, ) aws_assume_role_arn: Optional[str] = Field( + default=None, title="Assume Role ARN", description="If provided, the ARN of the role that should be assumed to connect to s3.", ) - _required_components: List[str] = ["auth_method"] + _required_components: ClassVar[List[str]] = ["auth_method"] - @root_validator(pre=True) - @classmethod - def keys_provided_if_needed(cls, values: Dict) -> Dict: + @model_validator(mode="after") + def keys_provided_if_needed(self) -> "S3Schema": """ Validates that both access and secret access keys are provided if using a `secret_keys` auth method. """ - if values.get("auth_method") == AWSAuthMethod.SECRET_KEYS.value and not ( - values.get("aws_access_key_id") and values.get("aws_secret_access_key") + if self.auth_method == AWSAuthMethod.SECRET_KEYS.value and not ( + self.aws_access_key_id and self.aws_secret_access_key ): raise ValueError( f"An Access Key ID and a Secret Access Key must be provided if using the `{AWSAuthMethod.SECRET_KEYS.value}` Authentication Method" ) - return values + return self class S3DocsSchema(S3Schema, NoValidationSchema): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_saas.py b/src/fides/api/schemas/connection_configuration/connection_secrets_saas.py index f1d7418a40..09375f434b 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_saas.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_saas.py @@ -1,8 +1,15 @@ import abc -from typing import Any, Dict, List, Type +from typing import Any, Dict, List, Optional, Type, Union from fideslang.models import FidesDatasetReference -from pydantic import BaseModel, Extra, Field, PrivateAttr, create_model, root_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PrivateAttr, + create_model, + model_validator, +) from pydantic.fields import FieldInfo from sqlalchemy.orm import Session @@ -20,16 +27,15 @@ class SaaSSchema(BaseModel, abc.ABC): Fields are added during runtime based on the connector_params and any external_references in the passed in saas_config""" - @root_validator + @model_validator(mode="before") @classmethod - def required_components_supplied( # type: ignore - cls, values: Dict[str, Any] - ) -> Dict[str, Any]: + def required_components_supplied(cls, values: Dict) -> Dict[str, Any]: # type: ignore """Validate that the minimum required components have been supplied.""" - # check required components are present required_components = [ - name for name, attributes in cls.__fields__.items() if attributes.required + name + for name, attributes in cls.model_fields.items() + if attributes.is_required() ] min_fields_present = all( values.get(component) for component in required_components @@ -69,6 +75,10 @@ def required_components_supplied( # type: ignore @classmethod def get_connector_param(cls, name: str) -> Dict[str, Any]: + if not cls.__private_attributes__: + # Not sure why this was needed for Pydantic V2. + # This was to address 'NoneType' object has no attribute 'default' + return {} return cls.__private_attributes__.get("_connector_params").default.get(name) # type: ignore @classmethod @@ -79,14 +89,9 @@ def external_references(cls) -> List[str]: if "external_reference" in property and property["external_reference"] ] - class Config: - """ - Certain SaaS workflows need to save secrets that are not part of the schema, - such as access and refresh tokens for OAuth2. So we allow extra fields - """ - - extra = Extra.ignore - orm_mode = True + model_config = ConfigDict( + extra="allow", from_attributes=True, hide_input_in_errors=True + ) class SaaSSchemaFactory: @@ -105,11 +110,16 @@ def get_saas_schema(self) -> Type[SaaSSchema]: for connector_param in self.saas_config.connector_params: param_type = list if connector_param.multiselect else str field_definitions[connector_param.name] = ( - Field( - title=connector_param.label, - description=connector_param.description, - default=connector_param.default_value, - sensitive=connector_param.sensitive, + ( + Optional[ + Union[str, List[str], int, List[int]] + ], # This matches the type of ConnectorParams.default_value + Field( + title=connector_param.label, + description=connector_param.description, + default=connector_param.default_value, + json_schema_extra={"sensitive": connector_param.sensitive}, + ), ) if connector_param.default_value else ( @@ -117,7 +127,7 @@ def get_saas_schema(self) -> Type[SaaSSchema]: FieldInfo( title=connector_param.label, description=connector_param.description, - sensitive=connector_param.sensitive, + json_schema_extra={"sensitive": connector_param.sensitive}, ), ) ) @@ -128,7 +138,9 @@ def get_saas_schema(self) -> Type[SaaSSchema]: FieldInfo( title=external_reference.label, description=external_reference.description, - external_reference=True, # metadata added so we can identify these secret schema fields as external references + json_schema_extra={ + "external_reference": True + }, # metadata added so we can identify these secret schema fields as external references ), ) SaaSSchema.__doc__ = f"{str(self.saas_config.type).capitalize()} secrets schema" # Dynamically override the docstring to create a description diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_scylla.py b/src/fides/api/schemas/connection_configuration/connection_secrets_scylla.py index c95d2eec9e..65a43ed26e 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_scylla.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_scylla.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import ClassVar, List, Optional from pydantic import Field @@ -27,14 +27,15 @@ class ScyllaSchema(ConnectionConfigSecretsSchema): password: str = Field( title="Password", description="The password used to authenticate and access the database.", - sensitive=True, + json_schema_extra={"sensitive": True}, ) keyspace: Optional[str] = Field( + default=None, title="Keyspace", description="The keyspace used. If not provided, DSRs for this integration will error. If the integration is used for D & D, then setting a keyspace is not required.", ) - _required_components: List[str] = [ + _required_components: ClassVar[List[str]] = [ "host", "username", "password", diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_snowflake.py b/src/fides/api/schemas/connection_configuration/connection_secrets_snowflake.py index 0564e3bc8d..b08fa01367 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_snowflake.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_snowflake.py @@ -1,6 +1,6 @@ -from typing import List, Optional +from typing import ClassVar, List, Optional -from pydantic import Field, root_validator +from pydantic import Field, model_validator from fides.api.schemas.base_class import NoValidationSchema from fides.api.schemas.connection_configuration.connection_secrets import ( @@ -31,19 +31,19 @@ class SnowflakeSchema(ConnectionConfigSecretsSchema): title="Password", description="The password used to authenticate and access the database. You can use a password or a private key, but not both.", default=None, - sensitive=True, + json_schema_extra={"sensitive": True}, ) private_key: Optional[str] = Field( title="Private key", description="The private key used to authenticate and access the database. If a `private_key_passphrase` is also provided, it is assumed to be encrypted; otherwise, it is assumed to be unencrypted.", default=None, - sensitive=True, + json_schema_extra={"sensitive": True}, ) private_key_passphrase: Optional[str] = Field( title="Passphrase", description="The passphrase used for the encrypted private key.", default=None, - sensitive=True, + json_schema_extra={"sensitive": True}, ) warehouse_name: str = Field( title="Warehouse", @@ -63,7 +63,7 @@ class SnowflakeSchema(ConnectionConfigSecretsSchema): description="The Snowflake role to assume for the session, if different than Username.", ) - _required_components: List[str] = [ + _required_components: ClassVar[List[str]] = [ "account_identifier", "user_login_name", "warehouse_name", @@ -71,25 +71,25 @@ class SnowflakeSchema(ConnectionConfigSecretsSchema): "schema_name", ] - @root_validator() - def validate_private_key_and_password(cls, values: dict) -> dict: - private_key: str = values.get("private_key", "") + @model_validator(mode="after") + def validate_private_key_and_password(self) -> "SnowflakeSchema": + private_key: str = self.private_key or "" - if values.get("password") and private_key: + if self.password and private_key: raise ValueError( "Cannot provide both password and private key at the same time." ) - if not any([values.get("password"), private_key]): + if not any([self.password, private_key]): raise ValueError("Must provide either a password or a private key.") if private_key: try: - values["private_key"] = format_private_key(private_key) + self.private_key = format_private_key(private_key) except IndexError: raise ValueError("Invalid private key format") - return values + return self class SnowflakeDocsSchema(SnowflakeSchema, NoValidationSchema): diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_sovrn.py b/src/fides/api/schemas/connection_configuration/connection_secrets_sovrn.py index ba7974e403..f26764ee7e 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_sovrn.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_sovrn.py @@ -20,10 +20,12 @@ class SovrnSchema(ExtendedEmailSchema): """ third_party_vendor_name: str = "Sovrn" - recipient_email_address: EmailStr = EmailStr("privacy@sovrn.com") - advanced_settings = AdvancedSettingsWithExtendedIdentityTypes( - identity_types=ExtendedIdentityTypes( - email=False, phone_number=False, cookie_ids=[SOVRN_REQUIRED_IDENTITY] + recipient_email_address: EmailStr = "privacy@sovrn.com" + advanced_settings: AdvancedSettingsWithExtendedIdentityTypes = ( + AdvancedSettingsWithExtendedIdentityTypes( + identity_types=ExtendedIdentityTypes( + email=False, phone_number=False, cookie_ids=[SOVRN_REQUIRED_IDENTITY] + ) ) ) diff --git a/src/fides/api/schemas/connection_configuration/connection_type_system_map.py b/src/fides/api/schemas/connection_configuration/connection_type_system_map.py index a94afefc9e..310d60e959 100644 --- a/src/fides/api/schemas/connection_configuration/connection_type_system_map.py +++ b/src/fides/api/schemas/connection_configuration/connection_type_system_map.py @@ -1,6 +1,6 @@ from typing import List, Optional, Union -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from fides.api.models.connectionconfig import ConnectionType from fides.api.schemas.connection_configuration.enums.system_type import SystemType @@ -15,13 +15,8 @@ class ConnectionSystemTypeMap(BaseModel): identifier: Union[ConnectionType, str] type: SystemType human_readable: str - encoded_icon: Optional[str] + encoded_icon: Optional[str] = None authorization_required: Optional[bool] = False - user_guide: Optional[str] + user_guide: Optional[str] = None supported_actions: List[ActionType] - - class Config: - """Use enum values and set orm mode""" - - use_enum_values = True - orm_mode = True + model_config = ConfigDict(use_enum_values=True, from_attributes=True) diff --git a/src/fides/api/schemas/connection_configuration/connections_secrets_https.py b/src/fides/api/schemas/connection_configuration/connections_secrets_https.py index 77fdb107b7..4eaafcad6f 100644 --- a/src/fides/api/schemas/connection_configuration/connections_secrets_https.py +++ b/src/fides/api/schemas/connection_configuration/connections_secrets_https.py @@ -1,4 +1,4 @@ -from typing import List +from typing import ClassVar, List from pydantic import Field @@ -12,9 +12,9 @@ class HttpsSchema(ConnectionConfigSecretsSchema): """Schema to validate the secrets needed to connect to a client api""" url: str - authorization: str = Field(sensitive=True) + authorization: str = Field(json_schema_extra={"sensitive": True}) - _required_components: List[str] = ["url", "authorization"] + _required_components: ClassVar[List[str]] = ["url", "authorization"] class HttpsDocsSchema(HttpsSchema, NoValidationSchema): diff --git a/src/fides/api/schemas/connection_configuration/saas_config_template_values.py b/src/fides/api/schemas/connection_configuration/saas_config_template_values.py index f8485a5d1d..c171fbff01 100644 --- a/src/fides/api/schemas/connection_configuration/saas_config_template_values.py +++ b/src/fides/api/schemas/connection_configuration/saas_config_template_values.py @@ -1,7 +1,7 @@ from typing import Any, Dict, Optional from fideslang.validation import FidesKey -from pydantic import BaseModel, Extra +from pydantic import BaseModel, ConfigDict from fides.api.models.connectionconfig import AccessLevel, ConnectionType from fides.api.schemas.connection_configuration import connection_secrets_schemas @@ -10,14 +10,12 @@ class SaasConnectionTemplateValues(BaseModel): """Schema with values to create both a Saas ConnectionConfig and DatasetConfig from a template""" - name: Optional[str] # For ConnectionConfig - key: Optional[FidesKey] # For ConnectionConfig - description: Optional[str] # For ConnectionConfig + name: Optional[str] = None # For ConnectionConfig + key: Optional[FidesKey] = None # For ConnectionConfig + description: Optional[str] = None # For ConnectionConfig secrets: connection_secrets_schemas # For ConnectionConfig instance_key: FidesKey # For DatasetConfig.fides_key - - class Config: - extra = Extra.ignore + model_config = ConfigDict(extra="ignore") def generate_config_data_from_template( self, config_from_template: Dict diff --git a/src/fides/api/schemas/consentable_item.py b/src/fides/api/schemas/consentable_item.py index 1c178adfe3..b4f1808913 100644 --- a/src/fides/api/schemas/consentable_item.py +++ b/src/fides/api/schemas/consentable_item.py @@ -72,7 +72,7 @@ def build_consent_item_hierarchy( """ return [ - ConsentableItem.from_orm(item) + ConsentableItem.model_validate(item) for item in consentable_items if item.parent_id is None ] diff --git a/src/fides/api/schemas/dataset.py b/src/fides/api/schemas/dataset.py index cb6b3d7962..d749cdc8d0 100644 --- a/src/fides/api/schemas/dataset.py +++ b/src/fides/api/schemas/dataset.py @@ -3,7 +3,7 @@ from fideslang.models import Dataset, DatasetCollection, DatasetField from fideslang.validation import FidesKey from loguru import logger -from pydantic import BaseModel, validator +from pydantic import BaseModel, ConfigDict, field_validator from fides.api import common_exceptions from fides.api.schemas.api import BulkResponse, BulkUpdateFailed @@ -30,7 +30,8 @@ def validate_data_categories_against_db( ] class DataCategoryValidationMixin(BaseModel): - @validator("data_categories", check_fields=False, allow_reuse=True) + @field_validator("data_categories", check_fields=False) + @classmethod def valid_data_categories( cls: Type["DataCategoryValidationMixin"], v: Optional[List[FidesKey]] ) -> Optional[List[FidesKey]]: @@ -38,9 +39,9 @@ def valid_data_categories( return _valid_data_categories(v, defined_data_categories) class FieldDataCategoryValidation(DatasetField, DataCategoryValidationMixin): - fields: Optional[List["FieldDataCategoryValidation"]] # type: ignore[assignment] + fields: Optional[List["FieldDataCategoryValidation"]] = None # type: ignore[assignment] - FieldDataCategoryValidation.update_forward_refs() + FieldDataCategoryValidation.model_rebuild() class CollectionDataCategoryValidation( DatasetCollection, DataCategoryValidationMixin @@ -50,7 +51,7 @@ class CollectionDataCategoryValidation( class DatasetDataCategoryValidation(Dataset, DataCategoryValidationMixin): collections: Sequence[CollectionDataCategoryValidation] # type: ignore[assignment] - DatasetDataCategoryValidation(**dataset.dict()) + DatasetDataCategoryValidation(**dataset.model_dump(mode="json")) def _valid_data_categories( @@ -106,11 +107,7 @@ class DatasetConfigSchema(FidesSchema): fides_key: FidesKey ctl_dataset: Dataset - - class Config: - """Set ORM Mode to True.""" - - orm_mode = True + model_config = ConfigDict(from_attributes=True) class BulkPutDataset(BulkResponse): diff --git a/src/fides/api/schemas/drp_privacy_request.py b/src/fides/api/schemas/drp_privacy_request.py index 7b9af3c0f2..616d2de056 100644 --- a/src/fides/api/schemas/drp_privacy_request.py +++ b/src/fides/api/schemas/drp_privacy_request.py @@ -1,7 +1,7 @@ from enum import Enum from typing import List, Optional -from pydantic import EmailStr, validator +from pydantic import ConfigDict, EmailStr, field_validator from fides.api.custom_types import PhoneNumber from fides.api.schemas.base_class import FidesSchema @@ -26,18 +26,15 @@ class DrpPrivacyRequestCreate(FidesSchema): """Data required to create a DRP PrivacyRequest""" meta: DrpMeta - regime: Optional[DrpRegime] + regime: Optional[DrpRegime] = None exercise: List[DrpAction] - relationships: Optional[List[str]] + relationships: Optional[List[str]] = None identity: str - status_callback: Optional[str] + status_callback: Optional[str] = None + model_config = ConfigDict(use_enum_values=True) - class Config: - """Populate models with the raw value of enum fields, rather than the enum itself""" - - use_enum_values = True - - @validator("exercise") + @field_validator("exercise") + @classmethod def check_exercise_length(cls, exercise: List[DrpAction]) -> List[DrpAction]: """Validate the only one exercise action is provided""" if len(exercise) > 1: @@ -48,29 +45,29 @@ def check_exercise_length(cls, exercise: List[DrpAction]) -> List[DrpAction]: class DrpIdentity(FidesSchema): """Drp identity props""" - aud: Optional[str] - sub: Optional[str] - name: Optional[str] - email: Optional[EmailStr] - email_verified: Optional[bool] - phone_number: Optional[PhoneNumber] - phone_number_verified: Optional[bool] - address: Optional[str] - address_verified: Optional[bool] - owner_of_attorney: Optional[str] + aud: Optional[str] = None + sub: Optional[str] = None + name: Optional[str] = None + email: Optional[EmailStr] = None + email_verified: Optional[bool] = None + phone_number: Optional[PhoneNumber] = None + phone_number_verified: Optional[bool] = None + address: Optional[str] = None + address_verified: Optional[bool] = None + owner_of_attorney: Optional[str] = None class DrpDataRightsResponse(FidesSchema): """Drp data rights response""" version: str - api_base: Optional[str] + api_base: Optional[str] = None actions: List[DrpAction] - user_relationships: Optional[List[str]] + user_relationships: Optional[List[str]] = None class DrpRevokeRequest(FidesSchema): """DRP Data Rights Revoke Request Body""" request_id: str - reason: Optional[str] + reason: Optional[str] = None diff --git a/src/fides/api/schemas/encryption_request.py b/src/fides/api/schemas/encryption_request.py index 00eed863a1..cbfe637fb2 100644 --- a/src/fides/api/schemas/encryption_request.py +++ b/src/fides/api/schemas/encryption_request.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from fides.api.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key from fides.config import CONFIG @@ -10,7 +10,8 @@ class AesEncryptionRequest(BaseModel): value: str key: str - @validator("key") + @field_validator("key") + @classmethod def validate_key(cls, v: str) -> bytes: """Convert string into bytes and verify this is the correct length""" diff --git a/src/fides/api/schemas/external_https.py b/src/fides/api/schemas/external_https.py index 1f2151c54f..41c0a94667 100644 --- a/src/fides/api/schemas/external_https.py +++ b/src/fides/api/schemas/external_https.py @@ -1,6 +1,6 @@ from typing import List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from fides.api.schemas.redis_cache import Identity @@ -13,22 +13,14 @@ class SecondPartyResponseFormat(BaseModel): derived_identity: Optional[Identity] = None halt: bool - - class Config: - """Using enum values""" - - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) class PrivacyRequestResumeFormat(BaseModel): """Expected request body to resume a privacy request after it was paused by a webhook""" derived_identity: Optional[Identity] = {} # type: ignore - - class Config: - """Using enum values""" - - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) class WebhookJWE(BaseModel): diff --git a/src/fides/api/schemas/language.py b/src/fides/api/schemas/language.py index 31045052ef..a4c315e72b 100644 --- a/src/fides/api/schemas/language.py +++ b/src/fides/api/schemas/language.py @@ -27,7 +27,7 @@ def _load_supported_languages() -> Dict[str, Language]: _languages = yaml.safe_load(file).get("languages", []) language_dict = {} for language in _languages: - language_dict[language["id"]] = Language.parse_obj(language) + language_dict[language["id"]] = Language.model_validate(language) return language_dict diff --git a/src/fides/api/schemas/limiter/rate_limit_config.py b/src/fides/api/schemas/limiter/rate_limit_config.py index 959423a95f..6b34e5353b 100644 --- a/src/fides/api/schemas/limiter/rate_limit_config.py +++ b/src/fides/api/schemas/limiter/rate_limit_config.py @@ -1,7 +1,7 @@ from enum import Enum -from typing import Dict, List, Optional +from typing import List, Optional -from pydantic import BaseModel, root_validator, validator +from pydantic import BaseModel, field_validator, model_validator class RateLimitPeriod(str, Enum): @@ -22,9 +22,10 @@ class RateLimit(BaseModel): rate: int period: RateLimitPeriod - custom_key: Optional[str] + custom_key: Optional[str] = None - @validator("rate") + @field_validator("rate") + @classmethod def rate_more_than_zero(cls, v: int) -> int: assert v > 0, "rate must be more than zero" return v @@ -35,13 +36,13 @@ class RateLimitConfig(BaseModel): A config object which allows configuring rate limits for connectors """ - limits: Optional[List[RateLimit]] + limits: Optional[List[RateLimit]] = None enabled: Optional[bool] = True - @root_validator - def validate_all(cls, values: Dict) -> Dict: - limits: Optional[List[RateLimit]] = values["limits"] - enabled: Optional[bool] = values["enabled"] + @model_validator(mode="after") + def validate_all(self) -> "RateLimitConfig": + limits: Optional[List[RateLimit]] = self.limits + enabled: Optional[bool] = self.enabled if enabled: assert ( @@ -49,4 +50,4 @@ def validate_all(cls, values: Dict) -> Dict: ), "limits must be set if rate limiter is enabled" if not enabled: assert not limits, "limits cannot be set if enabled is false" - return values + return self diff --git a/src/fides/api/schemas/manual_webhook_schemas.py b/src/fides/api/schemas/manual_webhook_schemas.py index 220b7044db..f2dc8842a0 100644 --- a/src/fides/api/schemas/manual_webhook_schemas.py +++ b/src/fides/api/schemas/manual_webhook_schemas.py @@ -1,7 +1,7 @@ -from typing import TYPE_CHECKING, List, Optional, Set +from typing import TYPE_CHECKING, Annotated, List, Optional, Set from fideslang.validation import FidesKey -from pydantic import ConstrainedStr, conlist, validator +from pydantic import ConfigDict, Field, StringConstraints, field_validator from fides.api.schemas.base_class import FidesSchema from fides.api.schemas.connection_configuration.connection_config import ( @@ -9,30 +9,22 @@ ) from fides.api.util.text import to_snake_case - -class PIIFieldType(ConstrainedStr): - """Using ConstrainedStr instead of constr to keep mypy happy""" - - min_length = 1 - max_length = 200 - strip_whitespace = True - - -class DSRLabelFieldType(ConstrainedStr): - """Using ConstrainedStr instead of constr to keep mypy happy""" - - max_length = 200 - strip_whitespace = True +DSRLabelFieldType = Annotated[ + str, StringConstraints(max_length=200, strip_whitespace=True) +] class ManualWebhookField(FidesSchema): """Schema to describe the attributes on a manual webhook field""" - pii_field: PIIFieldType + pii_field: Annotated[ + str, StringConstraints(min_length=1, max_length=200, strip_whitespace=True) + ] dsr_package_label: Optional[DSRLabelFieldType] = None data_categories: Optional[List[FidesKey]] = None - @validator("dsr_package_label") + @field_validator("dsr_package_label") + @classmethod def convert_empty_string_dsr_package_label( cls, value: Optional[str] ) -> Optional[str]: @@ -42,25 +34,23 @@ def convert_empty_string_dsr_package_label( """ return None if value == "" else value - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) if TYPE_CHECKING: ManualWebhookFieldsList = List[ManualWebhookField] else: - ManualWebhookFieldsList = conlist(ManualWebhookField, min_items=1) + ManualWebhookFieldsList = Annotated[List[ManualWebhookField], Field(min_length=1)] class AccessManualWebhooks(FidesSchema): """Expected request body for creating Access Manual Webhooks""" fields: ManualWebhookFieldsList + model_config = ConfigDict(from_attributes=True) - class Config: - orm_mode = True - - @validator("fields") + @field_validator("fields") + @classmethod def check_for_duplicates( cls, value: List[ManualWebhookField] ) -> List[ManualWebhookField]: @@ -89,7 +79,8 @@ def check_for_duplicates( return value - @validator("fields") + @field_validator("fields") + @classmethod def fields_must_exist( cls, value: List[ManualWebhookField] ) -> List[ManualWebhookField]: @@ -122,6 +113,4 @@ class AccessManualWebhookResponse(AccessManualWebhooks): connection_config: ConnectionConfigurationResponse id: str - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/src/fides/api/schemas/masking/masking_api.py b/src/fides/api/schemas/masking/masking_api.py index 3fe25f5aa3..16d87382cd 100644 --- a/src/fides/api/schemas/masking/masking_api.py +++ b/src/fides/api/schemas/masking/masking_api.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Union -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, model_validator from fides.api.schemas.policy import PolicyMaskingSpec @@ -12,7 +12,8 @@ class MaskingAPIRequest(BaseModel): masking_strategy: Union[PolicyMaskingSpec, List[PolicyMaskingSpec]] masking_strategies: List[PolicyMaskingSpec] = [] - @root_validator(pre=False) + @model_validator(mode="before") + @classmethod def build_masking_strategies(cls, values: Dict) -> Dict: """ Update "masking_strategies" field by inspecting "masking_strategy". diff --git a/src/fides/api/schemas/messaging/messaging.py b/src/fides/api/schemas/messaging/messaging.py index 121e36752c..18f4e5ac09 100644 --- a/src/fides/api/schemas/messaging/messaging.py +++ b/src/fides/api/schemas/messaging/messaging.py @@ -5,7 +5,7 @@ from fideslang.default_taxonomy import DEFAULT_TAXONOMY from fideslang.validation import FidesKey -from pydantic import BaseModel, Extra, Field, root_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator from fides.api.custom_types import PhoneNumber, SafeStr from fides.api.schemas import Msg @@ -114,7 +114,7 @@ class AccessRequestCompleteBodyParams(BaseModel): class RequestReviewDenyBodyParams(BaseModel): """Body params required for privacy request review deny template""" - rejection_reason: Optional[SafeStr] + rejection_reason: Optional[SafeStr] = None class ConsentPreferencesByUser(BaseModel): @@ -132,7 +132,8 @@ class ConsentPreferencesByUser(BaseModel): MinimalPrivacyPreferenceHistorySchema ] # Privacy preferences for new workflow - @root_validator + @model_validator(mode="before") + @classmethod def transform_data_use_format(cls, values: Dict[str, Any]) -> Dict[str, Any]: """Transform a data use fides_key to a corresponding name if possible""" consent_preferences = values.get("consent_preferences") or [] @@ -140,7 +141,8 @@ def transform_data_use_format(cls, values: Dict[str, Any]) -> Dict[str, Any]: preference.data_use = next( ( data_use.name - for data_use in DEFAULT_TAXONOMY.data_use or [] + for data_use in DEFAULT_TAXONOMY.data_use # pylint:disable=not-an-iterable + or [] if data_use.fides_key == preference.data_use ), preference.data_use, @@ -175,8 +177,6 @@ class UserInviteBodyParams(BaseModel): class FidesopsMessage( BaseModel, - smart_union=True, - arbitrary_types_allowed=True, ): """A mapping of action_type to body_params""" @@ -189,8 +189,12 @@ class FidesopsMessage( RequestReviewDenyBodyParams, AccessRequestCompleteBodyParams, ErasureRequestBodyParams, + ErrorNotificationBodyParams, + UserInviteBodyParams, ] - ] + ] = None + + model_config = ConfigDict(arbitrary_types_allowed=True) class EmailForActionType(BaseModel): @@ -224,11 +228,7 @@ class MessagingServiceDetailsMailchimpTransactional(BaseModel): """The details required to represent a Mailchimp Transactional email configuration.""" email_from: str - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class MessagingServiceDetailsMailgun(BaseModel): @@ -237,22 +237,14 @@ class MessagingServiceDetailsMailgun(BaseModel): is_eu_domain: Optional[bool] = False api_version: Optional[str] = "v3" domain: str - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class MessagingServiceDetailsTwilioEmail(BaseModel): """The details required to represent a Twilio email configuration.""" twilio_email_from: str - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class MessagingServiceSecrets(Enum): @@ -278,22 +270,14 @@ class MessagingServiceSecretsMailchimpTransactional(BaseModel): """The secrets required to connect to Mailchimp Transactional.""" mailchimp_transactional_api_key: str - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class MessagingServiceSecretsMailgun(BaseModel): """The secrets required to connect to Mailgun.""" mailgun_api_key: str - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class MessagingServiceSecretsTwilioSMS(BaseModel): @@ -301,15 +285,12 @@ class MessagingServiceSecretsTwilioSMS(BaseModel): twilio_account_sid: str twilio_auth_token: str - twilio_messaging_service_sid: Optional[str] - twilio_sender_phone_number: Optional[PhoneNumber] - - class Config: - """Restrict adding other fields through this schema.""" + twilio_messaging_service_sid: Optional[str] = None + twilio_sender_phone_number: Optional[PhoneNumber] = None + model_config = ConfigDict(extra="forbid") - extra = Extra.forbid - - @root_validator + @model_validator(mode="before") + @classmethod def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: sender_phone = values.get("twilio_sender_phone_number") if not values.get("twilio_messaging_service_sid") and not sender_phone: @@ -323,11 +304,7 @@ class MessagingServiceSecretsTwilioEmail(BaseModel): """The secrets required to connect to twilio email.""" twilio_api_key: str - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class MessagingConfigBase(BaseModel): @@ -340,18 +317,17 @@ class MessagingConfigBase(BaseModel): MessagingServiceDetailsTwilioEmail, MessagingServiceDetailsMailchimpTransactional, ] - ] - - class Config: - use_enum_values = False - orm_mode = True - extra = Extra.forbid + ] = None + model_config = ConfigDict( + use_enum_values=False, from_attributes=True, extra="forbid" + ) class MessagingConfigRequestBase(MessagingConfigBase): """Base model shared by messaging config requests to provide validation on request inputs""" - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: service_type = values.get("service_type") if service_type: @@ -385,7 +361,7 @@ class MessagingConfigRequest(MessagingConfigRequestBase): """Messaging Config Request Schema""" name: str - key: Optional[FidesKey] + key: Optional[FidesKey] = None class MessagingConfigResponse(MessagingConfigBase): @@ -393,10 +369,7 @@ class MessagingConfigResponse(MessagingConfigBase): name: str key: FidesKey - - class Config: - orm_mode = True - use_enum_values = True + model_config = ConfigDict(from_attributes=True, use_enum_values=True) SUPPORTED_MESSAGING_SERVICE_SECRETS = Union[ @@ -439,10 +412,12 @@ class MessagingConfigStatusMessage(BaseModel): class BasicMessagingTemplateBase(BaseModel): type: str content: Dict[str, Any] = Field( - example={ - "subject": "Message subject", - "body": "Custom message body", - } + examples=[ + { + "subject": "Message subject", + "body": "Custom message body", + } + ] ) @@ -467,62 +442,66 @@ class MessagingTemplateWithPropertiesBase(BaseModel): id: str type: str is_enabled: bool - properties: Optional[List[MinimalProperty]] + properties: Optional[List[MinimalProperty]] = None - class Config: - orm_mode = True - use_enum_values = True + model_config = ConfigDict(from_attributes=True, use_enum_values=True) class MessagingTemplateDefault(BaseModel): type: str is_enabled: bool content: Dict[str, Any] = Field( - example={ - "subject": "Message subject", - "body": "Custom message body", - } + examples=[ + { + "subject": "Message subject", + "body": "Custom message body", + } + ] ) class MessagingTemplateWithPropertiesSummary(MessagingTemplateWithPropertiesBase): - class Config: - orm_mode = True - use_enum_values = True + + model_config = ConfigDict(from_attributes=True, use_enum_values=True) class MessagingTemplateWithPropertiesDetail(MessagingTemplateWithPropertiesBase): content: Dict[str, Any] = Field( - example={ - "subject": "Message subject", - "body": "Custom message body", - } + examples=[ + { + "subject": "Message subject", + "body": "Custom message body", + } + ] ) - class Config: - orm_mode = True - use_enum_values = True + model_config = ConfigDict(from_attributes=True, use_enum_values=True) class MessagingTemplateWithPropertiesBodyParams(BaseModel): content: Dict[str, Any] = Field( - example={ - "subject": "Message subject", - "body": "Custom message body", - } + examples=[ + { + "subject": "Message subject", + "body": "Custom message body", + } + ] ) - properties: Optional[List[str]] + properties: Optional[List[str]] = None is_enabled: bool class MessagingTemplateWithPropertiesPatchBodyParams(BaseModel): content: Optional[Dict[str, Any]] = Field( - example={ - "subject": "Message subject", - "body": "Custom message body", - } + None, + examples=[ + { + "subject": "Message subject", + "body": "Custom message body", + } + ], ) - properties: Optional[List[str]] - is_enabled: Optional[bool] + properties: Optional[List[str]] = None + is_enabled: Optional[bool] = None diff --git a/src/fides/api/schemas/policy.py b/src/fides/api/schemas/policy.py index ba2f958971..c45c8c6a87 100644 --- a/src/fides/api/schemas/policy.py +++ b/src/fides/api/schemas/policy.py @@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional from fideslang.validation import FidesKey +from pydantic import ConfigDict from fides.api.schemas.api import BulkResponse, BulkUpdateFailed from fides.api.schemas.base_class import FidesSchema @@ -56,29 +57,21 @@ class PolicyMaskingSpecResponse(FidesSchema): class RuleTarget(FidesSchema): """An external representation of a Rule's target DataCategory within a Fidesops Policy""" - name: Optional[str] - key: Optional[FidesKey] + name: Optional[str] = None + key: Optional[FidesKey] = None # `data_category` is type str so that we can validate its contents against the DB records # outside of the schemas data_category: str - - class Config: - """Populate models with the raw value of enum fields, rather than the enum itself""" - - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) class RuleBase(FidesSchema): """An external representation of a Rule within a Fidesops Policy""" name: str - key: Optional[FidesKey] + key: Optional[FidesKey] = None action_type: ActionType - - class Config: - """Populate models with the raw value of enum fields, rather than the enum itself""" - - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) class RuleCreate(RuleBase): @@ -87,8 +80,8 @@ class RuleCreate(RuleBase): over a composite object. """ - storage_destination_key: Optional[FidesKey] - masking_strategy: Optional[PolicyMaskingSpec] + storage_destination_key: Optional[FidesKey] = None + masking_strategy: Optional[PolicyMaskingSpec] = None class RuleResponse(RuleBase): @@ -97,8 +90,8 @@ class RuleResponse(RuleBase): of the `PolicyMaskingSpec` that omits the configuration to avoid exposing secrets. """ - storage_destination: Optional[StorageDestinationResponse] - masking_strategy: Optional[PolicyMaskingSpecResponse] + storage_destination: Optional[StorageDestinationResponse] = None + masking_strategy: Optional[PolicyMaskingSpecResponse] = None class RuleResponseWithTargets(RuleBase): @@ -108,38 +101,33 @@ class RuleResponseWithTargets(RuleBase): configuration to avoid exposing secrets. """ - targets: Optional[List[RuleTarget]] - storage_destination: Optional[StorageDestinationResponse] - masking_strategy: Optional[PolicyMaskingSpecResponse] + targets: Optional[List[RuleTarget]] = None + storage_destination: Optional[StorageDestinationResponse] = None + masking_strategy: Optional[PolicyMaskingSpecResponse] = None class Rule(RuleBase): """A representation of a Rule that features all storage destination data.""" - storage_destination: Optional[StorageDestinationResponse] - masking_strategy: Optional[PolicyMaskingSpec] + storage_destination: Optional[StorageDestinationResponse] = None + masking_strategy: Optional[PolicyMaskingSpec] = None class Policy(FidesSchema): """An external representation of a Fidesops Policy""" name: str - key: Optional[FidesKey] - drp_action: Optional[DrpAction] - execution_timeframe: Optional[int] - - class Config: - """Populate models with the raw value of enum fields, rather than the enum itself""" - - use_enum_values = True - orm_mode = True + key: Optional[FidesKey] = None + drp_action: Optional[DrpAction] = None + execution_timeframe: Optional[int] = None + model_config = ConfigDict(use_enum_values=True, from_attributes=True) class PolicyResponse(Policy): """A holistic view of a Policy record, including all foreign keys by default.""" - rules: Optional[List[RuleResponse]] - drp_action: Optional[DrpAction] + rules: Optional[List[RuleResponse]] = None + drp_action: Optional[DrpAction] = None class BulkPutRuleTargetResponse(BulkResponse): diff --git a/src/fides/api/schemas/policy_webhooks.py b/src/fides/api/schemas/policy_webhooks.py index c827bce0b8..c1f3e7c86a 100644 --- a/src/fides/api/schemas/policy_webhooks.py +++ b/src/fides/api/schemas/policy_webhooks.py @@ -1,6 +1,7 @@ from typing import List, Optional from fideslang.validation import FidesKey +from pydantic import ConfigDict from fides.api.models.policy import WebhookDirection from fides.api.schemas.base_class import FidesSchema @@ -13,47 +14,35 @@ class WebhookBase(FidesSchema): """Base schema for Webhooks""" direction: WebhookDirection - key: Optional[FidesKey] - name: Optional[str] + key: Optional[FidesKey] = None + name: Optional[str] = None class PolicyWebhookCreate(WebhookBase): """Request schema for creating/updating a Policy Webhook""" connection_config_key: FidesKey - - class Config: - """Populate models with the raw value of enum fields, rather than the enum itself""" - - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) class PolicyWebhookResponse(WebhookBase): """Response schema after creating a PolicyWebhook""" - connection_config: Optional[ConnectionConfigurationResponse] + connection_config: Optional[ConnectionConfigurationResponse] = None order: int - - class Config: - """Set orm_mode to True""" - - orm_mode = True + model_config = ConfigDict(from_attributes=True) class PolicyWebhookUpdate(FidesSchema): """Request schema for updating a single webhook - fields are optional""" - direction: Optional[WebhookDirection] - name: Optional[str] - connection_config_key: Optional[FidesKey] - order: Optional[int] - - class Config: - """Only the included attributes will be used""" - - orm_mode = True - extra = "forbid" - use_enum_values = True + direction: Optional[WebhookDirection] = None + name: Optional[str] = None + connection_config_key: Optional[FidesKey] = None + order: Optional[int] = None + model_config = ConfigDict( + from_attributes=True, extra="forbid", use_enum_values=True + ) class WebhookOrder(FidesSchema): @@ -61,11 +50,7 @@ class WebhookOrder(FidesSchema): key: FidesKey order: int - - class Config: - """Set orm_mode to True""" - - orm_mode = True + model_config = ConfigDict(from_attributes=True) class PolicyWebhookUpdateResponse(FidesSchema): @@ -81,8 +66,4 @@ class PolicyWebhookDeleteResponse(FidesSchema): """Response schema after deleting a webhook; new_order includes remaining reordered webhooks if applicable""" new_order: List[WebhookOrder] - - class Config: - """Set orm_mode to True""" - - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/src/fides/api/schemas/pre_approval_webhooks.py b/src/fides/api/schemas/pre_approval_webhooks.py index 5f29aa1279..5b9903a89c 100644 --- a/src/fides/api/schemas/pre_approval_webhooks.py +++ b/src/fides/api/schemas/pre_approval_webhooks.py @@ -1,6 +1,7 @@ from typing import Optional from fideslang.validation import FidesKey +from pydantic import ConfigDict from fides.api.schemas.base_class import FidesSchema from fides.api.schemas.connection_configuration.connection_config import ( @@ -11,7 +12,7 @@ class WebhookBase(FidesSchema): """Base schema for Webhooks""" - key: Optional[FidesKey] + key: Optional[FidesKey] = None name: str @@ -19,33 +20,21 @@ class PreApprovalWebhookCreate(WebhookBase): """Request schema for creating/updating a Pre Approval Webhook""" connection_config_key: FidesKey - - class Config: - """Populate models with the raw value of enum fields, rather than the enum itself""" - - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) class PreApprovalWebhookResponse(WebhookBase): """Response schema after creating/updating/getting a PreApprovalWebhook""" - connection_config: Optional[ConnectionConfigurationResponse] - - class Config: - """Set orm_mode to True""" - - orm_mode = True + connection_config: Optional[ConnectionConfigurationResponse] = None + model_config = ConfigDict(from_attributes=True) class PreApprovalWebhookUpdate(FidesSchema): """Request schema for updating a single webhook - fields are optional""" - name: Optional[str] - connection_config_key: Optional[FidesKey] - - class Config: - """Only the included attributes will be used""" - - orm_mode = True - extra = "forbid" - use_enum_values = True + name: Optional[str] = None + connection_config_key: Optional[FidesKey] = None + model_config = ConfigDict( + from_attributes=True, extra="forbid", use_enum_values=True + ) diff --git a/src/fides/api/schemas/privacy_center_config.py b/src/fides/api/schemas/privacy_center_config.py index 0777e59c62..26fd06c3c8 100644 --- a/src/fides/api/schemas/privacy_center_config.py +++ b/src/fides/api/schemas/privacy_center_config.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Literal, Optional, Union -from pydantic import Extra, Field, root_validator, validator +from pydantic import ConfigDict, Field, field_validator, model_validator from fides.api.schemas.base_class import FidesSchema @@ -15,15 +15,11 @@ class IdentityInputs(FidesSchema): name: Optional[RequiredType] = None email: Optional[RequiredType] = None phone: Optional[RequiredType] = None - - class Config: - """Allows extra fields to be provided but they must have a value of type CustomIdentity.""" - - extra = Extra.allow + model_config = ConfigDict(extra="allow") def __init__(self, **data: Any): for field, value in data.items(): - if field not in self.__fields__: + if field not in self.model_fields: if isinstance(value, CustomIdentity): data[field] = value elif isinstance(value, dict) and "label" in value: @@ -43,7 +39,8 @@ class CustomPrivacyRequestField(FidesSchema): hidden: Optional[bool] = False query_param_key: Optional[str] = None - @root_validator + @model_validator(mode="before") + @classmethod def validate_default_value(cls, values: Dict[str, Any]) -> Dict[str, Any]: if ( values.get("hidden") @@ -61,23 +58,23 @@ class PrivacyRequestOption(FidesSchema): icon_path: str title: str description: str - description_subtext: Optional[List[str]] - confirm_button_text: Optional[str] = Field(alias="confirmButtonText") - cancel_button_text: Optional[str] = Field(alias="cancelButtonText") + description_subtext: Optional[List[str]] = None + confirm_button_text: Optional[str] = Field(alias="confirmButtonText", default=None) + cancel_button_text: Optional[str] = Field(alias="cancelButtonText", default=None) identity_inputs: Optional[IdentityInputs] = None custom_privacy_request_fields: Optional[Dict[str, CustomPrivacyRequestField]] = None class ConsentConfigButton(FidesSchema): description: str - description_subtext: Optional[List[str]] - confirm_button_text: Optional[str] = Field(alias="confirmButtonText") - cancel_button_text: Optional[str] = Field(alias="cancelButtonText") + description_subtext: Optional[List[str]] = None + confirm_button_text: Optional[str] = Field(alias="confirmButtonText", default=None) + cancel_button_text: Optional[str] = Field(alias="cancelButtonText", default=None) icon_path: str identity_inputs: IdentityInputs custom_privacy_request_fields: Optional[Dict[str, CustomPrivacyRequestField]] = None title: str - modal_title: Optional[str] = Field(alias="modalTitle") + modal_title: Optional[str] = Field(alias="modalTitle", default=None) class ConditionalValue(FidesSchema): @@ -87,23 +84,24 @@ class ConditionalValue(FidesSchema): class ConfigConsentOption(FidesSchema): cookie_keys: List[str] = Field([], alias="cookieKeys") - default: Optional[Union[bool, ConditionalValue]] + default: Optional[Union[bool, ConditionalValue]] = None description: str fides_data_use_key: str = Field(alias="fidesDataUseKey") - highlight: Optional[bool] + highlight: Optional[bool] = None name: str url: str - executable: Optional[bool] + executable: Optional[bool] = None class ConsentConfigPage(FidesSchema): consent_options: List[ConfigConsentOption] = Field([], alias="consentOptions") description: str - description_subtext: Optional[List[str]] - policy_key: Optional[str] + description_subtext: Optional[List[str]] = None + policy_key: Optional[str] = None title: str - @validator("consent_options") + @field_validator("consent_options") + @classmethod def validate_consent_options( cls, consent_options: List[ConfigConsentOption] ) -> List[ConfigConsentOption]: @@ -128,18 +126,18 @@ class PrivacyCenterConfig(FidesSchema): title: str description: str - description_subtext: Optional[List[str]] - addendum: Optional[List[str]] - server_url_development: Optional[str] - server_url_production: Optional[str] - logo_path: Optional[str] - logo_url: Optional[str] - favicon_path: Optional[str] + description_subtext: Optional[List[str]] = None + addendum: Optional[List[str]] = None + server_url_development: Optional[str] = None + server_url_production: Optional[str] = None + logo_path: Optional[str] = None + logo_url: Optional[str] = None + favicon_path: Optional[str] = None actions: List[PrivacyRequestOption] - include_consent: Optional[bool] = Field(alias="includeConsent") + include_consent: Optional[bool] = Field(alias="includeConsent", default=None) consent: ConsentConfig - privacy_policy_url: Optional[str] - privacy_policy_url_text: Optional[str] + privacy_policy_url: Optional[str] = None + privacy_policy_url_text: Optional[str] = None class PartialPrivacyRequestOption(FidesSchema): diff --git a/src/fides/api/schemas/privacy_notice.py b/src/fides/api/schemas/privacy_notice.py index 82bfdb6b97..cca96e8acb 100644 --- a/src/fides/api/schemas/privacy_notice.py +++ b/src/fides/api/schemas/privacy_notice.py @@ -1,6 +1,7 @@ from typing import List, Optional from fideslang.validation import FidesKey +from pydantic import ConfigDict from fides.api.models.privacy_notice import ConsentMechanism, EnforcementLevel from fides.api.schemas.base_class import FidesSchema @@ -22,7 +23,4 @@ class PrivacyNoticeHistorySchema(FidesSchema): enforcement_level: Optional[EnforcementLevel] version: float translation_id: Optional[str] - - class Config: - use_enum_values = True - orm_mode = True + model_config = ConfigDict(use_enum_values=True, from_attributes=True) diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py index 89e9bae7b1..a9036db8e9 100644 --- a/src/fides/api/schemas/privacy_request.py +++ b/src/fides/api/schemas/privacy_request.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Type, Union from fideslang.validation import FidesKey -from pydantic import Extra, Field, validator +from pydantic import ConfigDict, Field, field_serializer, field_validator from fides.api.custom_types import SafeStr from fides.api.models.audit_log import AuditLogAction @@ -40,17 +40,12 @@ class PrivacyRequestDRPStatusResponse(FidesSchema): request_id: str received_at: datetime - expected_by: Optional[datetime] - processing_details: Optional[str] + expected_by: Optional[datetime] = None + processing_details: Optional[str] = None status: PrivacyRequestDRPStatus - reason: Optional[str] - user_verification_url: Optional[str] - - class Config: - """Set orm_mode and use_enum_values""" - - orm_mode = True - use_enum_values = True + reason: Optional[str] = None + user_verification_url: Optional[str] = None + model_config = ConfigDict(from_attributes=True, use_enum_values=True) class Consent(FidesSchema): @@ -79,10 +74,10 @@ class ConsentReport(Consent): class PrivacyRequestCreate(FidesSchema): """Data required to create a PrivacyRequest""" - external_id: Optional[str] - started_processing_at: Optional[datetime] - finished_processing_at: Optional[datetime] - requested_at: Optional[datetime] + external_id: Optional[str] = None + started_processing_at: Optional[datetime] = None + finished_processing_at: Optional[datetime] = None + requested_at: Optional[datetime] = None identity: Identity consent_request_id: Optional[str] = None custom_privacy_request_fields: Optional[Dict[str, CustomPrivacyRequestField]] = None @@ -91,7 +86,8 @@ class PrivacyRequestCreate(FidesSchema): property_id: Optional[str] = None consent_preferences: Optional[List[Consent]] = None # TODO Slated for deprecation - @validator("encryption_key") + @field_validator("encryption_key") + @classmethod def validate_encryption_key( cls: Type["PrivacyRequestCreate"], value: Optional[str] = None ) -> Optional[str]: @@ -106,7 +102,7 @@ class ConsentRequestCreate(FidesSchema): identity: Identity custom_privacy_request_fields: Optional[Dict[str, CustomPrivacyRequestField]] = None - property_id: Optional[str] + property_id: Optional[str] = None class FieldsAffectedResponse(FidesSchema): @@ -115,45 +111,32 @@ class FieldsAffectedResponse(FidesSchema): path: Optional[str] field_name: Optional[str] data_categories: Optional[List[str]] - - class Config: - """Set orm_mode and use_enum_values""" - - orm_mode = True - use_enum_values = True + model_config = ConfigDict(from_attributes=True, use_enum_values=True) class ExecutionLogStatusSerializeOverride(FidesSchema): """Override to serialize "paused" Execution Logs as awaiting_processing instead""" - class Config: - """Set orm_mode and use_enum_values""" - - orm_mode = True - use_enum_values = False # Used in conjunction with the "dict" override below - - def dict(self, *args: Any, **kwargs: Any) -> Dict: - """ - When serializing, use the Execution Log Status name instead of the value + status: ExecutionLogStatus + model_config = ConfigDict(from_attributes=True, use_enum_values=False) - This is because our "awaiting_processing" status is "paused" in the db, - but we want to use "awaiting_processing" everywhere in the app. + @field_serializer("status") + def serialize_status(self, status: ExecutionLogStatus) -> str: + """For statuses, we want to use the name instead of the value + This is for backwards compatibility where we are repurposing the "paused" status + to read "awaiting processing" """ - data = super().dict(*args, **kwargs) - if isinstance(data.get("status"), ExecutionLogStatus): - data["status"] = data["status"].name - return data + return status.name class ExecutionLogResponse(ExecutionLogStatusSerializeOverride): """Schema for the embedded ExecutionLogs associated with a PrivacyRequest""" - collection_name: Optional[str] - fields_affected: Optional[List[FieldsAffectedResponse]] - message: Optional[str] + collection_name: Optional[str] = None + fields_affected: Optional[List[FieldsAffectedResponse]] = None + message: Optional[str] = None action_type: ActionType - status: ExecutionLogStatus - updated_at: Optional[datetime] + updated_at: Optional[datetime] = None class PrivacyRequestTaskSchema(ExecutionLogStatusSerializeOverride): @@ -161,7 +144,6 @@ class PrivacyRequestTaskSchema(ExecutionLogStatusSerializeOverride): id: str collection_address: str - status: ExecutionLogStatus created_at: datetime updated_at: datetime upstream_tasks: List[str] @@ -172,27 +154,39 @@ class PrivacyRequestTaskSchema(ExecutionLogStatusSerializeOverride): class ExecutionLogDetailResponse(ExecutionLogResponse): """Schema for the detailed ExecutionLogs when accessed directly""" - connection_key: Optional[str] - dataset_name: Optional[str] + connection_key: Optional[str] = None + dataset_name: Optional[str] = None -class ExecutionAndAuditLogResponse(ExecutionLogStatusSerializeOverride): +class ExecutionAndAuditLogResponse(FidesSchema): """Schema for the combined ExecutionLogs and Audit Logs associated with a PrivacyRequest""" - connection_key: Optional[str] - collection_name: Optional[str] - fields_affected: Optional[List[FieldsAffectedResponse]] - message: Optional[str] - action_type: Optional[ActionType] - status: Optional[Union[ExecutionLogStatus, AuditLogAction]] - updated_at: Optional[datetime] - user_id: Optional[str] + connection_key: Optional[str] = None + collection_name: Optional[str] = None + fields_affected: Optional[List[FieldsAffectedResponse]] = None + message: Optional[str] = None + action_type: Optional[ActionType] = None + status: Optional[Union[ExecutionLogStatus, AuditLogAction, str]] = None + updated_at: Optional[datetime] = None + user_id: Optional[str] = None + model_config = ConfigDict(populate_by_name=True) + + @field_serializer("status") + def serialize_status( + self, status: Optional[Union[ExecutionLogStatus, AuditLogAction, str]] + ) -> Optional[str]: + """For statuses, we want to use the name instead of the value + This is for backwards compatibility where we are repurposing the "paused" status + to read "awaiting processing" - class Config: - """Set orm_mode and allow population by field name""" + Generally, status will be a string here because we had to convert both ExecutionLogStatuses + and AuditLogAction statuses to strings so we could union both resources into the same query + """ + if isinstance(status, (AuditLogAction, ExecutionLogStatus)): + return status.name if status else None - allow_population_by_field_name = True + return "awaiting_processing" if status == "paused" else status class RowCountRequest(FidesSchema): @@ -225,34 +219,30 @@ class PrivacyRequestResponse(FidesSchema): """Schema to check the status of a PrivacyRequest""" id: str - created_at: Optional[datetime] - started_processing_at: Optional[datetime] - reviewed_at: Optional[datetime] - reviewed_by: Optional[str] - reviewer: Optional[PrivacyRequestReviewer] - finished_processing_at: Optional[datetime] - identity_verified_at: Optional[datetime] - paused_at: Optional[datetime] + created_at: Optional[datetime] = None + started_processing_at: Optional[datetime] = None + reviewed_at: Optional[datetime] = None + reviewed_by: Optional[str] = None + reviewer: Optional[PrivacyRequestReviewer] = None + finished_processing_at: Optional[datetime] = None + identity_verified_at: Optional[datetime] = None + paused_at: Optional[datetime] = None status: PrivacyRequestStatus - external_id: Optional[str] + external_id: Optional[str] = None # This field intentionally doesn't use the Identity schema # as it is an API response field, and we don't want to reveal any more # about our PII structure than is explicitly stored in the cache on request # creation. - identity: Optional[Dict[str, Union[Optional[str], Dict[str, Any]]]] - custom_privacy_request_fields: Optional[Dict[str, Any]] + identity: Optional[Dict[str, Union[Optional[str], Dict[str, Any]]]] = None + custom_privacy_request_fields: Optional[Dict[str, Any]] = None policy: PolicySchema action_required_details: Optional[CheckpointActionRequiredDetails] = None - resume_endpoint: Optional[str] - days_left: Optional[int] - custom_privacy_request_fields_approved_by: Optional[str] - custom_privacy_request_fields_approved_at: Optional[datetime] - - class Config: - """Set orm_mode and use_enum_values""" + resume_endpoint: Optional[str] = None + days_left: Optional[int] = None + custom_privacy_request_fields_approved_by: Optional[str] = None + custom_privacy_request_fields_approved_at: Optional[datetime] = None - orm_mode = True - use_enum_values = True + model_config = ConfigDict(from_attributes=True, use_enum_values=True) class PrivacyRequestVerboseResponse(PrivacyRequestResponse): @@ -262,23 +252,19 @@ class PrivacyRequestVerboseResponse(PrivacyRequestResponse): execution_and_audit_logs_by_dataset: Dict[ str, List[ExecutionAndAuditLogResponse] ] = Field(alias="results") - - class Config: - """Allow the results field to be populated by the 'PrivacyRequest.execution_logs_by_dataset' property""" - - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class ReviewPrivacyRequestIds(FidesSchema): """Pass in a list of privacy request ids""" - request_ids: List[str] = Field(..., max_items=50) + request_ids: List[str] = Field(..., max_length=50) class DenyPrivacyRequests(ReviewPrivacyRequestIds): """Pass in a list of privacy request ids and rejection reason""" - reason: Optional[SafeStr] + reason: Optional[SafeStr] = None class BulkPostPrivacyRequests(BulkResponse): @@ -308,11 +294,11 @@ class ConsentWithExecutableStatus(FidesSchema): class ConsentPreferencesWithVerificationCode(FidesSchema): """Schema for consent preferences including the verification code.""" - code: Optional[str] + code: Optional[str] = None consent: List[Consent] policy_key: Optional[FidesKey] = None - executable_options: Optional[List[ConsentWithExecutableStatus]] - browser_identity: Optional[Identity] + executable_options: Optional[List[ConsentWithExecutableStatus]] = None + browser_identity: Optional[Identity] = None class ConsentRequestResponse(FidesSchema): @@ -345,10 +331,10 @@ class RequestTaskCallbackRequest(FidesSchema): class PrivacyRequestFilter(FidesSchema): request_id: Optional[str] = None identities: Optional[Dict[str, Any]] = Field( - None, example={"email": "user@example.com", "loyalty_id": "CH-1"} + None, examples=[{"email": "user@example.com", "loyalty_id": "CH-1"}] ) custom_privacy_request_fields: Optional[Dict[str, Any]] = Field( - None, example={"site_id": "abc", "subscriber_id": "123"} + None, examples=[{"site_id": "abc", "subscriber_id": "123"}] ) status: Optional[Union[PrivacyRequestStatus, List[PrivacyRequestStatus]]] = None created_lt: Optional[datetime] = None @@ -368,10 +354,10 @@ class PrivacyRequestFilter(FidesSchema): sort_field: str = "created_at" sort_direction: ColumnSort = ColumnSort.DESC - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") - @validator("status") + @field_validator("status") + @classmethod def validate_status_field( cls, field_value: Union[PrivacyRequestStatus, List[PrivacyRequestStatus]], diff --git a/src/fides/api/schemas/property.py b/src/fides/api/schemas/property.py index 8eac0b5a15..7165611942 100644 --- a/src/fides/api/schemas/property.py +++ b/src/fides/api/schemas/property.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Any, Iterable, List, Optional -from pydantic import validator +from pydantic import field_validator from fides.api.custom_types import CssStr from fides.api.schemas.base_class import FidesSchema @@ -65,12 +65,13 @@ class PublicPropertyResponse(FidesSchema): type: PropertyType id: Optional[str] = None experiences: List[MinimalPrivacyExperienceConfig] - messaging_templates: Optional[List[MinimalMessagingTemplate]] - privacy_center_config: Optional[PrivacyCenterConfig] - stylesheet: Optional[CssStr] + messaging_templates: Optional[List[MinimalMessagingTemplate]] = None + privacy_center_config: Optional[PrivacyCenterConfig] = None + stylesheet: Optional[CssStr] = None paths: List[str] - @validator("paths", pre=True) + @field_validator("paths", mode="before") + @classmethod def convert_to_list(cls, value: Any) -> Any: # type: ignore[misc] """ Convert the 'paths' value to a list if it is an iterable of strings. diff --git a/src/fides/api/schemas/redis_cache.py b/src/fides/api/schemas/redis_cache.py index 2844976f21..2561bb5ed0 100644 --- a/src/fides/api/schemas/redis_cache.py +++ b/src/fides/api/schemas/redis_cache.py @@ -1,7 +1,7 @@ import uuid from typing import Any, Dict, List, Optional, Union -from pydantic import EmailStr, Extra, Field, StrictInt, StrictStr, validator +from pydantic import ConfigDict, EmailStr, Field, StrictInt, StrictStr, field_validator from fides.api.custom_types import PhoneNumber from fides.api.schemas.base_class import FidesSchema @@ -14,11 +14,7 @@ class IdentityBase(FidesSchema): phone_number: Optional[PhoneNumber] = None email: Optional[EmailStr] = None - - class Config: - """Only allow phone_number, and email.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class LabeledIdentity(FidesSchema): @@ -34,21 +30,20 @@ class Identity(IdentityBase): """Some PII grouping pertaining to a human""" # These are repeated so we can continue to forbid extra fields - phone_number: Optional[PhoneNumber] = Field(None, title="Phone number") - email: Optional[EmailStr] = Field(None, title="Email") - ga_client_id: Optional[str] = Field(None, title="GA client ID") - ljt_readerID: Optional[str] = Field(None, title="LJT reader ID") - fides_user_device_id: Optional[str] = Field(None, title="Fides user device ID") - external_id: Optional[str] = Field(None, title="External ID") + phone_number: Optional[PhoneNumber] = Field(default=None, title="Phone number") + email: Optional[EmailStr] = Field(default=None, title="Email") + ga_client_id: Optional[str] = Field(default=None, title="GA client ID") + ljt_readerID: Optional[str] = Field(default=None, title="LJT reader ID") + fides_user_device_id: Optional[str] = Field( + default=None, title="Fides user device ID" + ) + external_id: Optional[str] = Field(default=None, title="External ID") - class Config: - """Allows extra fields to be provided but they must have a value of type LabeledIdentity.""" - - extra = Extra.allow + model_config = ConfigDict(extra="allow") def __init__(self, **data: Any): for field, value in data.items(): - if field not in self.__fields__: + if field not in self.model_fields: if isinstance(value, LabeledIdentity): data[field] = value elif isinstance(value, dict) and "label" in value and "value" in value: @@ -60,7 +55,7 @@ def __init__(self, **data: Any): ) super().__init__(**data) - @validator("fides_user_device_id") + @field_validator("fides_user_device_id") @classmethod def validate_fides_user_device_id(cls, v: Optional[str]) -> Optional[str]: """Validate the uuid format of the fides user device id while still keeping the data type a string""" @@ -74,32 +69,50 @@ def dict(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: Returns a dictionary with LabeledIdentity values returned as simple values. """ d = super().dict(*args, **kwargs) - for key, value in self.__dict__.items(): - if isinstance(value, LabeledIdentity): - d[key] = value.value + for key, value in d.items(): + if key in self.model_fields: + d[key] = value else: + # Turn LabeledIdentity into simple values + # 'customer_id': {'label': 'Customer ID', 'value': '123'} -> 'customer_id': '123' + d[key] = value.get("value") + return d + + def model_dump(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Returns a dictionary with LabeledIdentity values returned as simple values. + In pydantic v2, model_dump is preferred over dict + """ + d = super().model_dump(*args, **kwargs) + for key, value in d.items(): + if key in self.model_fields: d[key] = value + else: + # Turn LabeledIdentity into simple values + # 'customer_id': {'label': 'Customer ID', 'value': '123'} -> 'customer_id': '123' + d[key] = value.get("value") return d def labeled_dict( self, include_default_labels: Optional[bool] = False ) -> Dict[str, Any]: """Returns a dictionary that preserves the labels for all custom/labeled identities.""" - d = {} - for key, value in self.__dict__.items(): - if key in self.__fields__: - if include_default_labels: - d[key] = { - "label": self.__fields__[key].field_info.title, - "value": value, - } - else: - d[key] = value + d = super().model_dump() + for field, _ in self.model_fields.items(): + value = getattr(self, field, None) + if include_default_labels: + d[field] = { + "label": self.model_fields[field].title, + "value": value, + } else: - if isinstance(value, LabeledIdentity): - d[key] = value.dict() - else: - d[key] = value + d[field] = value + for field in self.__pydantic_extra__ or {}: # pylint:disable=not-an-iterable + value = getattr(self, field, None) + if isinstance(value, LabeledIdentity): + d[field] = value.model_dump(mode="json") + else: + d[field] = value return d diff --git a/src/fides/api/schemas/registration.py b/src/fides/api/schemas/registration.py index c43710cf9d..9221e129ee 100644 --- a/src/fides/api/schemas/registration.py +++ b/src/fides/api/schemas/registration.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import EmailStr, Extra +from pydantic import ConfigDict, EmailStr from fides.api.schemas.base_class import FidesSchema @@ -11,11 +11,7 @@ class GetRegistrationStatusResponse(FidesSchema): """ opt_in: bool = False - - class Config: - """Only allow the `opt_in` status of this registration to be returned.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class Registration(GetRegistrationStatusResponse): diff --git a/src/fides/api/schemas/saas/connector_template.py b/src/fides/api/schemas/saas/connector_template.py index 6cec2450aa..1258127ec2 100644 --- a/src/fides/api/schemas/saas/connector_template.py +++ b/src/fides/api/schemas/saas/connector_template.py @@ -1,7 +1,7 @@ from typing import List, Optional from fideslang.models import Dataset -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from fides.api.schemas.policy import ActionType from fides.api.schemas.saas.saas_config import SaaSConfig @@ -16,23 +16,25 @@ class ConnectorTemplate(BaseModel): config: str dataset: str - icon: Optional[str] + icon: Optional[str] = None human_readable: str authorization_required: bool - user_guide: Optional[str] + user_guide: Optional[str] = None supported_actions: List[ActionType] - @validator("config") - def validate_config(cls, config: str) -> str: + @field_validator("config") + @classmethod + def validate_config(cls, value: str) -> str: """Validates the config at the given path""" - saas_config = SaaSConfig(**load_config_from_string(config)) + saas_config = SaaSConfig(**load_config_from_string(value)) if saas_config.fides_key != "": raise ValueError( "Hard-coded fides_key detected in the config, replace all instances of it with " ) - return config + return value - @validator("dataset") + @field_validator("dataset") + @classmethod def validate_dataset(cls, dataset: str) -> str: """Validates the dataset at the given path""" saas_dataset = Dataset(**load_dataset_from_string(dataset)) diff --git a/src/fides/api/schemas/saas/saas_config.py b/src/fides/api/schemas/saas/saas_config.py index 0749dadbff..ad19f8dc37 100644 --- a/src/fides/api/schemas/saas/saas_config.py +++ b/src/fides/api/schemas/saas/saas_config.py @@ -3,7 +3,7 @@ from fideslang.models import FidesCollectionKey, FidesDatasetReference from fideslang.validation import FidesKey -from pydantic import BaseModel, Extra, root_validator, validator +from pydantic import BaseModel, ConfigDict, field_validator, model_validator from fides.api.common_exceptions import ValidationError from fides.api.graph.config import ( @@ -31,12 +31,13 @@ class ParamValue(BaseModel): """ name: str - identity: Optional[str] - references: Optional[List[Union[FidesDatasetReference, str]]] - connector_param: Optional[str] + identity: Optional[str] = None + references: Optional[List[Union[FidesDatasetReference, str]]] = None + connector_param: Optional[str] = None unpack: Optional[bool] = False - @validator("references") + @field_validator("references") + @classmethod def check_reference_direction( cls, references: Optional[List[Union[FidesDatasetReference, str]]] ) -> Optional[List[Union[FidesDatasetReference, str]]]: @@ -49,18 +50,18 @@ def check_reference_direction( ) return references - @root_validator - def check_exactly_one_value_field(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @model_validator(mode="after") + def check_exactly_one_value_field(self) -> "ParamValue": value_fields = [ - bool(values.get("identity")), - bool(values.get("references")), - bool(values.get("connector_param")), + bool(self.identity), + bool(self.references), + bool(self.connector_param), ] if sum(value_fields) != 1: raise ValueError( "Must have exactly one of 'identity', 'references', or 'connector_param'" ) - return values + return self class Strategy(BaseModel): @@ -75,7 +76,7 @@ class ClientConfig(BaseModel): protocol: str host: str - authentication: Optional[Strategy] + authentication: Optional[Strategy] = None class Header(BaseModel): @@ -110,34 +111,30 @@ class SaaSRequest(BaseModel): Includes optional strategies for postprocessing and pagination. """ - request_override: Optional[str] - path: Optional[str] - method: Optional[HTTPMethod] + request_override: Optional[str] = None + path: Optional[str] = None + method: Optional[HTTPMethod] = None headers: Optional[List[Header]] = [] query_params: Optional[List[QueryParam]] = [] - body: Optional[str] + body: Optional[str] = None param_values: Optional[List[ParamValue]] = [] - client_config: Optional[ClientConfig] - data_path: Optional[str] - postprocessors: Optional[List[Strategy]] - pagination: Optional[Strategy] + client_config: Optional[ClientConfig] = None + data_path: Optional[str] = None + postprocessors: Optional[List[Strategy]] = None + pagination: Optional[Strategy] = None grouped_inputs: Optional[List[str]] = [] ignore_errors: Optional[Union[bool, List[int]]] = False - rate_limit_config: Optional[RateLimitConfig] + rate_limit_config: Optional[RateLimitConfig] = None async_config: Optional[AsyncConfig] = None skip_missing_param_values: Optional[bool] = ( False # Skip instead of raising an exception if placeholders can't be populated in body ) + model_config = ConfigDict( + from_attributes=True, use_enum_values=True, extra="forbid" + ) - class Config: - """Populate models with the raw value of enum fields, rather than the enum itself""" - - orm_mode = True - use_enum_values = True - extra = Extra.forbid - - @root_validator(pre=True) - def validate_request_for_pagination(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @model_validator(mode="after") + def validate_request_for_pagination(self) -> "SaaSRequest": """ Calls the appropriate validation logic for the request based on the specified pagination strategy. Passes in the raw value dict @@ -149,21 +146,21 @@ def validate_request_for_pagination(cls, values: Dict[str, Any]) -> Dict[str, An PaginationStrategy, ) - pagination = values.get("pagination") + pagination = self.pagination if pagination is not None: pagination_strategy = PaginationStrategy.get_strategy( - pagination.get("strategy"), pagination.get("configuration") + pagination.strategy, pagination.configuration ) - pagination_strategy.validate_request(values) - return values + pagination_strategy.validate_request(self.model_dump(mode="json")) + return self - @root_validator - def validate_grouped_inputs(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @model_validator(mode="after") + def validate_grouped_inputs(self) -> "SaaSRequest": """Validate that grouped_inputs must reference fields from the same collection""" - grouped_inputs = set(values.get("grouped_inputs", [])) + grouped_inputs = set(self.grouped_inputs or []) if grouped_inputs: - param_values = values.get("param_values", []) + param_values = self.param_values or [] names = {param.name for param in param_values} if not grouped_inputs.issubset(names): @@ -196,17 +193,17 @@ def validate_grouped_inputs(cls, values: Dict[str, Any]) -> Dict[str, Any]: "Grouped input fields must all reference the same collection." ) - return values + return self - @root_validator - def validate_request(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @model_validator(mode="after") + def validate_request(self) -> "SaaSRequest": """Validate that configs related to request overrides are set properly""" - if not values.get("request_override"): - if not values.get("path"): + if not self.request_override: + if not self.path: raise ValueError( "A request must specify a path if no request_override is provided" ) - if not values.get("method"): + if not self.method: raise ValueError( "A request must specify a method if no request_override is provided" ) @@ -214,8 +211,8 @@ def validate_request(cls, values: Dict[str, Any]) -> Dict[str, Any]: else: # if a request override is specified, many fields are NOT allowed invalid = [ k - for k in values.keys() - if values.get(k) + for k in self.model_fields + if getattr(self, k) and k not in ("request_override", "param_values", "grouped_inputs") ] if invalid: @@ -224,7 +221,7 @@ def validate_request(cls, values: Dict[str, Any]) -> Dict[str, Any]: f"Invalid properties [{invalid_joined}] on a request with request_override specified" ) - return values + return self class ReadSaaSRequest(SaaSRequest): @@ -233,17 +230,17 @@ class ReadSaaSRequest(SaaSRequest): that is used to format each collection result. """ - output: Optional[str] + output: Optional[str] = None - @root_validator - def validate_request(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @model_validator(mode="after") + def validate_request(self) -> "ReadSaaSRequest": """Validate that configs related to read requests are set properly""" - if not values.get("request_override"): - if not (values.get("path") or values.get("output")): + if not self.request_override: + if not (self.path or self.output): raise ValueError( "A read request must specify either a path or an output if no request_override is provided" ) - if values.get("path") and not values.get("method"): + if self.path and not self.method: raise ValueError( "A read request must specify a method if a path is provided and no request_override is specified" ) @@ -254,22 +251,24 @@ def validate_request(cls, values: Dict[str, Any]) -> Dict[str, Any]: "grouped_inputs", } invalid = [ - k for k in values.keys() if values.get(k) and k not in allowed_fields + k + for k in self.model_fields + if getattr(self, k) and k not in allowed_fields ] if invalid: invalid_joined = ", ".join(invalid) raise ValueError( f"Invalid properties [{invalid_joined}] on a read request with request_override specified" ) - return values + return self class SaaSRequestMap(BaseModel): """A map of actions to SaaS requests""" read: Union[ReadSaaSRequest, List[ReadSaaSRequest]] = [] - update: Optional[SaaSRequest] - delete: Optional[SaaSRequest] + update: Optional[SaaSRequest] = None + delete: Optional[SaaSRequest] = None class ConsentRequestMap(BaseModel): @@ -278,7 +277,8 @@ class ConsentRequestMap(BaseModel): opt_in: Union[SaaSRequest, List[SaaSRequest]] = [] opt_out: Union[SaaSRequest, List[SaaSRequest]] = [] - @validator("opt_in", "opt_out") + @field_validator("opt_in", "opt_out") + @classmethod def validate_list_field( cls, field_value: Union[SaaSRequest, List[SaaSRequest]], @@ -302,7 +302,8 @@ class Endpoint(BaseModel): after: List[FidesCollectionKey] = [] erase_after: List[FidesCollectionKey] = [] - @validator("requests") + @field_validator("requests") + @classmethod def validate_grouped_inputs( cls, requests: SaaSRequestMap, @@ -325,14 +326,18 @@ class ConnectorParam(BaseModel): """Used to define the required parameters for the connector (user and constants)""" name: str - label: Optional[str] - options: Optional[List[str]] # list of possible values for the connector param - default_value: Optional[Union[str, List[str]]] + label: Optional[str] = None + options: Optional[List[str]] = ( + None # list of possible values for the connector param + ) + # If you change the default_value type, update this in SaaSSchemaFactory.get_saas_schema + default_value: Optional[Union[str, List[str], int, List[int]]] = None multiselect: Optional[bool] = False - description: Optional[str] + description: Optional[str] = None sensitive: Optional[bool] = False - @root_validator + @model_validator(mode="before") + @classmethod def validate_connector_param(cls, values: Dict[str, Any]) -> Dict[str, Any]: """Verify the default_value is one of the values specified in the options list""" @@ -370,8 +375,8 @@ def validate_connector_param(cls, values: Dict[str, Any]) -> Dict[str, Any]: class ExternalDatasetReference(BaseModel): name: str - label: Optional[str] - description: Optional[str] + label: Optional[str] = None + description: Optional[str] = None class SaaSConfigBase(BaseModel): @@ -391,15 +396,13 @@ def fides_key_prop(self) -> FidesKey: def name_prop(self) -> str: return self.name - @validator("type", pre=True) + @field_validator("type", mode="before") + @classmethod def lowercase_saas_type(cls, value: str) -> str: """Enforce lowercase on saas type.""" return value.lower() - class Config: - """Populate models with the raw value of enum fields, rather than the enum itself""" - - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) class SaaSConfig(SaaSConfigBase): @@ -417,14 +420,14 @@ class SaaSConfig(SaaSConfigBase): version: str replaceable: bool = False connector_params: List[ConnectorParam] - external_references: Optional[List[ExternalDatasetReference]] + external_references: Optional[List[ExternalDatasetReference]] = None client_config: ClientConfig endpoints: List[Endpoint] test_request: SaaSRequest data_protection_request: Optional[SaaSRequest] = None # GDPR Delete - rate_limit_config: Optional[RateLimitConfig] - consent_requests: Optional[ConsentRequestMap] - user_guide: Optional[str] + rate_limit_config: Optional[RateLimitConfig] = None + consent_requests: Optional[ConsentRequestMap] = None + user_guide: Optional[str] = None @property def top_level_endpoint_dict(self) -> Dict[str, Endpoint]: @@ -507,7 +510,12 @@ def _process_param_values( references.append( ( FieldAddress(resolved_reference.dataset, first, *rest), - resolved_reference.direction, + ( + # Convert back into a literal for Pydantic V2 + resolved_reference.direction.value + if resolved_reference.direction + else resolved_reference.direction + ), ) ) fields.append(ScalarField(name=param.name, references=references)) @@ -533,7 +541,7 @@ def resolve_param_reference( raise ValidationError( f"External dataset reference with provided name {reference} not found in connector's secrets." ) - reference = FidesDatasetReference.parse_obj(secrets[reference]) + reference = FidesDatasetReference.model_validate(secrets[reference]) return reference @property @@ -579,7 +587,7 @@ class SaaSConfigValidationDetails(FidesSchema): Message with any validation issues with the SaaS config """ - msg: Optional[str] + msg: Optional[str] = None class ValidateSaaSConfigResponse(FidesSchema): diff --git a/src/fides/api/schemas/saas/shared_schemas.py b/src/fides/api/schemas/saas/shared_schemas.py index 26b7207d7c..0dc94402f2 100644 --- a/src/fides/api/schemas/saas/shared_schemas.py +++ b/src/fides/api/schemas/saas/shared_schemas.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Any, Dict, Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class HTTPMethod(Enum): @@ -25,18 +25,14 @@ class SaaSRequestParams(BaseModel): path: str headers: Dict[str, Any] = {} query_params: Dict[str, Any] = {} - body: Optional[str] - - class Config: - """Using enum values""" - - use_enum_values = True + body: Optional[str] = None + model_config = ConfigDict(use_enum_values=True) class ConnectorParamRef(BaseModel): """A reference to a value in the connector params (by name)""" - connector_param: Any + connector_param: Any = None class IdentityParamRef(BaseModel): diff --git a/src/fides/api/schemas/saas/strategy_configuration.py b/src/fides/api/schemas/saas/strategy_configuration.py index 1595a7dcad..bb2c672c53 100644 --- a/src/fides/api/schemas/saas/strategy_configuration.py +++ b/src/fides/api/schemas/saas/strategy_configuration.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Union -from pydantic import BaseModel, root_validator, validator +from pydantic import BaseModel, ConfigDict, field_validator, model_validator from fides.api.schemas.saas.saas_config import Header, QueryParam, SaaSRequest from fides.api.schemas.saas.shared_schemas import ConnectorParamRef, IdentityParamRef @@ -34,9 +34,10 @@ class OffsetPaginationConfiguration(StrategyConfiguration): incremental_param: str increment_by: int - limit: Optional[Union[int, ConnectorParamRef]] + limit: Optional[Union[int, ConnectorParamRef]] = None - @validator("increment_by") + @field_validator("increment_by") + @classmethod def check_increment_by(cls, increment_by: int) -> int: if increment_by == 0: raise ValueError("'increment_by' cannot be zero") @@ -56,10 +57,11 @@ class LinkPaginationConfiguration(StrategyConfiguration): """Gets the URL for the next page from the headers or the body.""" source: LinkSource - rel: Optional[str] - path: Optional[str] + rel: Optional[str] = None + path: Optional[str] = None - @root_validator + @model_validator(mode="before") + @classmethod def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: source = values.get("source") if source == LinkSource.headers.value and values.get("rel") is None: @@ -72,10 +74,7 @@ def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: ) return values - class Config: - """Using enum values""" - - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) class CursorPaginationConfiguration(StrategyConfiguration): @@ -92,11 +91,12 @@ class ApiKeyAuthenticationConfiguration(StrategyConfiguration): API key parameter to be added in as a header or query param """ - headers: Optional[List[Header]] - query_params: Optional[List[QueryParam]] - body: Optional[str] + headers: Optional[List[Header]] = None + query_params: Optional[List[QueryParam]] = None + body: Optional[str] = None - @root_validator + @model_validator(mode="before") + @classmethod def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: headers = values.get("headers") query_params = values.get("query_params") @@ -116,7 +116,7 @@ class BasicAuthenticationConfiguration(StrategyConfiguration): """ username: str - password: Optional[str] + password: Optional[str] = None class BearerAuthenticationConfiguration(StrategyConfiguration): @@ -143,9 +143,9 @@ class OAuth2BaseConfiguration(StrategyConfiguration): do not specify a TTL for the access tokens. """ - expires_in: Optional[int] + expires_in: Optional[int] = None token_request: SaaSRequest - refresh_request: Optional[SaaSRequest] + refresh_request: Optional[SaaSRequest] = None class OAuth2AuthorizationCodeConfiguration(OAuth2BaseConfiguration): diff --git a/src/fides/api/schemas/storage/storage.py b/src/fides/api/schemas/storage/storage.py index f2f36805f1..b2e87d658b 100644 --- a/src/fides/api/schemas/storage/storage.py +++ b/src/fides/api/schemas/storage/storage.py @@ -2,7 +2,14 @@ from typing import Any, Dict, List, Optional, Union from fideslang.validation import FidesKey -from pydantic import Extra, ValidationError, root_validator, validator +from pydantic import ( + ConfigDict, + Field, + ValidationError, + ValidationInfo, + field_validator, + model_validator, +) from pydantic.main import BaseModel from fides.api.schemas.api import BulkResponse, BulkUpdateFailed @@ -30,22 +37,14 @@ class StorageDetails(Enum): NAMING = "naming" MAX_RETRIES = "max_retries" AUTH_METHOD = "auth_method" - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class FileBasedStorageDetails(BaseModel): """A base class for all storage configuration that uses a file system.""" naming: str = FileNaming.request_id.value # How to name the uploaded file - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class AWSAuthMethod(str, Enum): @@ -59,9 +58,7 @@ class StorageDetailsS3(FileBasedStorageDetails): auth_method: AWSAuthMethod bucket: str max_retries: Optional[int] = 0 - - class Config: - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) class StorageDetailsLocal(FileBasedStorageDetails): @@ -79,10 +76,7 @@ class StorageSecrets(Enum): class StorageSecretsLocal(BaseModel): """A dummy schema for allowing any / no secrets for local filestorage.""" - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.allow + model_config = ConfigDict(extra="allow") class StorageSecretsS3(BaseModel): @@ -90,11 +84,7 @@ class StorageSecretsS3(BaseModel): aws_access_key_id: str aws_secret_access_key: str - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class StorageType(Enum): @@ -119,25 +109,23 @@ class StorageDestinationBase(BaseModel): details: Union[ StorageDetailsS3, StorageDetailsLocal, - ] + ] = Field(validate_default=True) format: Optional[ResponseFormat] = ResponseFormat.json.value # type: ignore + model_config = ConfigDict( + use_enum_values=True, from_attributes=True, extra="forbid" + ) - class Config: - use_enum_values = True - orm_mode = True - extra = Extra.forbid - - @validator("details", pre=True, always=True) + @field_validator("details", mode="before") @classmethod def validate_details_validator( cls, v: Dict[str, str], - values: Dict[str, Any], + info: ValidationInfo, ) -> Dict[str, str]: """ Custom validation logic for the `details` field. """ - storage_type = values.get("type") + storage_type = info.data.get("type") if not storage_type: raise ValueError("A `type` field must be specified.") @@ -164,7 +152,7 @@ def validate_details( f"`storage_type` {storage_type} has no supported `details` validation." ) try: - schema.parse_obj(details) # type: ignore + schema.model_validate(details) # type: ignore except ValidationError as exc: # Pydantic requires validators raise either a ValueError, TypeError, or AssertionError # so this exception is cast into a `ValueError`. @@ -173,15 +161,14 @@ def validate_details( return details - @root_validator - @classmethod - def format_validator(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @model_validator(mode="after") + def format_validator(self) -> "StorageDestinationBase": """ Custom validation to ensure that local destination formats are valid. """ restricted_destinations = [StorageType.local.value] - storage_type = values.get("type") - response_format = values.get("format") + storage_type = self.type + response_format = self.format if ( storage_type in restricted_destinations and response_format @@ -192,18 +179,15 @@ def format_validator(cls, values: Dict[str, Any]) -> Dict[str, Any]: "Only JSON or HTML upload format are supported for local storage destinations." ) - return values + return self class StorageDestination(StorageDestinationBase): """Storage Destination Schema""" name: str - key: Optional[FidesKey] - - class Config: - use_enum_values = True - orm_mode = True + key: Optional[FidesKey] = None + model_config = ConfigDict(use_enum_values=True, from_attributes=True) class StorageDestinationResponse(BaseModel): @@ -215,17 +199,14 @@ class StorageDestinationResponse(BaseModel): key: FidesKey format: ResponseFormat is_default: bool = False - - class Config: - orm_mode = True - use_enum_values = True + model_config = ConfigDict(from_attributes=True, use_enum_values=True) class BulkPutStorageConfigResponse(BulkResponse): """Schema with mixed success/failure responses for Bulk Create/Update of StorageConfig.""" - succeeded: List[StorageDestinationResponse] - failed: List[BulkUpdateFailed] + succeeded: List[StorageDestinationResponse] = [] + failed: List[BulkUpdateFailed] = [] SUPPORTED_STORAGE_SECRETS = StorageSecretsS3 diff --git a/src/fides/api/schemas/system.py b/src/fides/api/schemas/system.py index 278fc7ea76..05945c4995 100644 --- a/src/fides/api/schemas/system.py +++ b/src/fides/api/schemas/system.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional, Sequence from fideslang.models import Cookies, PrivacyDeclaration, System -from pydantic import Field +from pydantic import ConfigDict, Field from pydantic.main import BaseModel from fides.api.schemas.connection_configuration.connection_config import ( @@ -56,11 +56,9 @@ class SystemResponse(BasicSystemResponse): class SystemHistoryResponse(BaseModel): """Response schema for a single system history entry""" - edited_by: Optional[str] + edited_by: Optional[str] = None system_id: str before: Dict[str, Any] after: Dict[str, Any] created_at: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/src/fides/api/schemas/user.py b/src/fides/api/schemas/user.py index ad0d238685..1b1f1b738d 100644 --- a/src/fides/api/schemas/user.py +++ b/src/fides/api/schemas/user.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Optional -from pydantic import EmailStr, validator +from pydantic import EmailStr, field_validator from fides.api.cryptography.cryptographic_util import decode_password from fides.api.schemas.base_class import FidesSchema @@ -21,13 +21,13 @@ class UserCreate(FidesSchema): """Data required to create a FidesUser.""" username: str - password: Optional[str] + password: Optional[str] = None email_address: EmailStr - first_name: Optional[str] - last_name: Optional[str] + first_name: Optional[str] = None + last_name: Optional[str] = None disabled: bool = False - @validator("username") + @field_validator("username") @classmethod def validate_username(cls, username: str) -> str: """Ensure username does not have spaces.""" @@ -35,7 +35,7 @@ def validate_username(cls, username: str) -> str: raise ValueError("Usernames cannot have spaces.") return username - @validator("password") + @field_validator("password") @classmethod def validate_password(cls, password: str) -> str: """Add some password requirements""" @@ -69,7 +69,7 @@ class UserLogin(FidesSchema): username: str password: str - @validator("password") + @field_validator("password") @classmethod def validate_password(cls, password: str) -> str: """Convert b64 encoded password to normal string""" @@ -83,10 +83,10 @@ class UserResponse(FidesSchema): username: str created_at: datetime email_address: Optional[EmailStr] - first_name: Optional[str] - last_name: Optional[str] + first_name: Optional[str] = None + last_name: Optional[str] = None disabled: Optional[bool] = False - disabled_reason: Optional[str] + disabled_reason: Optional[str] = None class UserLoginResponse(FidesSchema): @@ -112,9 +112,9 @@ class UserForcePasswordReset(FidesSchema): class UserUpdate(FidesSchema): """Data required to update a FidesUser""" - email_address: Optional[EmailStr] - first_name: Optional[str] - last_name: Optional[str] + email_address: Optional[EmailStr] = None + first_name: Optional[str] = None + last_name: Optional[str] = None class DisabledReason(Enum): diff --git a/src/fides/api/schemas/user_permission.py b/src/fides/api/schemas/user_permission.py index d0e003267a..21e43533c6 100644 --- a/src/fides/api/schemas/user_permission.py +++ b/src/fides/api/schemas/user_permission.py @@ -1,6 +1,6 @@ from typing import List, Optional -from pydantic import validator +from pydantic import ConfigDict, field_validator from fides.api.oauth.roles import RoleRegistryEnum from fides.api.schemas.base_class import FidesSchema @@ -15,19 +15,15 @@ class UserPermissionsCreate(FidesSchema): """ roles: List[RoleRegistryEnum] - - class Config: - """So roles are strings when we add to the db""" - - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) class UserPermissionsEdit(UserPermissionsCreate): """Data required to edit a FidesUserPermissions record.""" - id: Optional[ - str - ] # I don't think this should be in the request body, so making it optional. + id: Optional[str] = ( + None # I don't think this should be in the request body, so making it optional. + ) class UserPermissionsResponse(UserPermissionsCreate): @@ -38,11 +34,10 @@ class UserPermissionsResponse(UserPermissionsCreate): total_scopes: List[ ScopeRegistryEnum ] # Returns a list of scopes inherited via roles + model_config = ConfigDict(use_enum_values=True) - class Config: - use_enum_values = True - - @validator("total_scopes", pre=True) + @field_validator("total_scopes", mode="before") + @classmethod def validate_obsolete_total_scopes( cls, total_scopes: List[ScopeRegistryEnum] ) -> List[ScopeRegistryEnum]: diff --git a/src/fides/api/service/connectors/consent_email_connector.py b/src/fides/api/service/connectors/consent_email_connector.py index 5287229684..37d01f57cd 100644 --- a/src/fides/api/service/connectors/consent_email_connector.py +++ b/src/fides/api/service/connectors/consent_email_connector.py @@ -235,7 +235,7 @@ def batch_email_send(self, privacy_requests: Query) -> None: filtered_privacy_request_schemas: List[ MinimalPrivacyPreferenceHistorySchema ] = [ - MinimalPrivacyPreferenceHistorySchema.from_orm(privacy_pref) + MinimalPrivacyPreferenceHistorySchema.model_validate(privacy_pref) for privacy_pref in filtered_privacy_preference_records ] diff --git a/src/fides/api/service/connectors/fides/fides_client.py b/src/fides/api/service/connectors/fides/fides_client.py index 9b3c9a0e56..a3e08a3af0 100644 --- a/src/fides/api/service/connectors/fides/fides_client.py +++ b/src/fides/api/service/connectors/fides/fides_client.py @@ -57,7 +57,8 @@ def login(self) -> None: ) try: response = httpx.post( - f"{self.uri}{urls.V1_URL_PREFIX}{urls.LOGIN}", json=ul.dict() + f"{self.uri}{urls.V1_URL_PREFIX}{urls.LOGIN}", + json=ul.model_dump(mode="json"), ) except RequestError as e: logger.error("Error logging in on remote Fides {}: {}", self.uri, str(e)) @@ -120,7 +121,7 @@ def create_privacy_request( request: Request = self.authenticated_request( method="POST", path=urls.V1_URL_PREFIX + urls.PRIVACY_REQUEST_AUTHENTICATED, - json=[pr.dict()], + json=[pr.model_dump(mode="json")], ) response = self.session.send(request) diff --git a/src/fides/api/service/connectors/fides_connector.py b/src/fides/api/service/connectors/fides_connector.py index e4027a4344..86812cb53e 100644 --- a/src/fides/api/service/connectors/fides_connector.py +++ b/src/fides/api/service/connectors/fides_connector.py @@ -48,6 +48,7 @@ def __init__(self, configuration: ConnectionConfig): def query_config(self, node: ExecutionNode) -> QueryConfig[Any]: """Return the query config that corresponds to this connector type""" # no query config for fides connectors + raise NotImplementedError() def create_client(self) -> FidesClient: """Returns a client used to connect to a Fides instance""" diff --git a/src/fides/api/service/connectors/http_connector.py b/src/fides/api/service/connectors/http_connector.py index 5f70fe24a4..50d4716928 100644 --- a/src/fides/api/service/connectors/http_connector.py +++ b/src/fides/api/service/connectors/http_connector.py @@ -68,6 +68,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: def query_config(self, node: ExecutionNode) -> QueryConfig[Any]: """Return the query config that corresponds to this connector type""" + raise NotImplementedError("Query config not implemented for HTTPS") def retrieve_data( self, diff --git a/src/fides/api/service/connectors/s3_connector.py b/src/fides/api/service/connectors/s3_connector.py index 2afc69fb99..35ed1b5157 100644 --- a/src/fides/api/service/connectors/s3_connector.py +++ b/src/fides/api/service/connectors/s3_connector.py @@ -25,8 +25,8 @@ def create_client(self) -> Any: # type: ignore """Returns a client for s3""" config = S3Schema(**self.configuration.secrets or {}) return get_aws_session( - auth_method=config.auth_method.value, - storage_secrets=config.dict(), # type: ignore + auth_method=config.auth_method.value, # pylint: disable=no-member + storage_secrets=config.model_dump(mode="json"), # type: ignore assume_role_arn=config.aws_assume_role_arn, ) @@ -57,6 +57,7 @@ def retrieve_data( input_data: Dict[str, List[Any]], ) -> List[Row]: """DSR execution not yet supported for S3""" + return [] def mask_data( self, @@ -67,6 +68,7 @@ def mask_data( rows: List[Row], ) -> int: """DSR execution not yet supported for S3""" + return 0 def close(self) -> None: """Close any held resources""" diff --git a/src/fides/api/service/connectors/saas/connector_registry_service.py b/src/fides/api/service/connectors/saas/connector_registry_service.py index 7799287aa4..81265a529c 100644 --- a/src/fides/api/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/service/connectors/saas/connector_registry_service.py @@ -388,7 +388,9 @@ def update_saas_configs(db: Session) -> None: conditions=(ConnectionConfig.saas_config["type"].astext == connector_type), ).all() for connection_config in connection_configs: - saas_config_instance = SaaSConfig.parse_obj(connection_config.saas_config) + saas_config_instance = SaaSConfig.model_validate( + connection_config.saas_config + ) if parse_version(saas_config_instance.version) < template_version: logger.info( "Updating SaaS config instance '{}' of type '{}' as its version, {}, was found to be lower than the template version {}", diff --git a/src/fides/api/service/connectors/scylla_connector.py b/src/fides/api/service/connectors/scylla_connector.py index 8733516994..43a821930c 100644 --- a/src/fides/api/service/connectors/scylla_connector.py +++ b/src/fides/api/service/connectors/scylla_connector.py @@ -30,8 +30,9 @@ class ScyllaConnector(BaseConnector[Cluster]): def build_uri(self) -> str: """ - Builds URI + Builds URI - Not yet implemented """ + return "" def get_config(self) -> ScyllaSchema: return ScyllaSchema(**self.configuration.secrets or {}) diff --git a/src/fides/api/service/connectors/sql_connector.py b/src/fides/api/service/connectors/sql_connector.py index e9e01953a0..7256493c83 100644 --- a/src/fides/api/service/connectors/sql_connector.py +++ b/src/fides/api/service/connectors/sql_connector.py @@ -483,7 +483,7 @@ def build_uri(self) -> str: """Build URI of format""" config = self.secrets_schema(**self.configuration.secrets or {}) dataset = f"/{config.dataset}" if config.dataset else "" - return f"bigquery://{config.keyfile_creds.project_id}{dataset}" + return f"bigquery://{config.keyfile_creds.project_id}{dataset}" # pylint: disable=no-member # Overrides SQLConnector.create_client def create_client(self) -> Engine: @@ -597,7 +597,7 @@ def get_connect_args(self) -> Dict[str, Any]: if config.private_key_passphrase: private_key_encoded = serialization.load_pem_private_key( config.private_key.encode(), - password=config.private_key_passphrase.encode(), + password=config.private_key_passphrase.encode(), # pylint: disable=no-member backend=default_backend(), ) private_key = private_key_encoded.private_bytes( diff --git a/src/fides/api/service/masking/strategy/masking_strategy.py b/src/fides/api/service/masking/strategy/masking_strategy.py index efc1bd7e0e..5a425d47d3 100644 --- a/src/fides/api/service/masking/strategy/masking_strategy.py +++ b/src/fides/api/service/masking/strategy/masking_strategy.py @@ -25,7 +25,8 @@ def secrets_required(self) -> bool: """Determines whether secrets are needed for specific masking strategy""" def generate_secrets_for_cache(self) -> List[MaskingSecretCache]: - """Generates secrets for strategy""" + """Generates secrets for strategy - optionally override on subclasses""" + return [] @classmethod @abstractmethod diff --git a/src/fides/api/service/messaging/message_dispatch_service.py b/src/fides/api/service/messaging/message_dispatch_service.py index 47e2b0e138..13489faabd 100644 --- a/src/fides/api/service/messaging/message_dispatch_service.py +++ b/src/fides/api/service/messaging/message_dispatch_service.py @@ -88,7 +88,7 @@ def check_and_dispatch_error_notifications(db: Session) -> None: body_params=ErrorNotificationBodyParams( unsent_errors=len(unsent_errors) ), - ).dict(), + ).model_dump(mode="json"), "service_type": config_proxy.notifications.notification_service_type, "to_identity": {"email": email}, "property_id": None, @@ -112,12 +112,12 @@ def dispatch_message_task( """ A wrapper function to dispatch a message task into the Celery queues """ - schema = FidesopsMessage.parse_obj(message_meta) + schema = FidesopsMessage.model_validate(message_meta) with self.get_new_session() as db: dispatch_message( db, schema.action_type, - Identity.parse_obj(to_identity), + Identity.model_validate(to_identity), service_type, schema.body_params, property_id, @@ -222,6 +222,7 @@ def dispatch_message( RequestReviewDenyBodyParams, ErasureRequestBodyParams, UserInviteBodyParams, + ErrorNotificationBodyParams, ] ] = None, subject_override: Optional[str] = None, @@ -560,7 +561,7 @@ def _mailchimp_transactional_dispatcher( data=data, ) if not response.ok: - logger.error("Email failed to send with status code: %s" % response.status_code) + logger.error("Email failed to send with status code: %s", response.status_code) raise MessageDispatchException( f"Email failed to send with status code {response.status_code}" ) diff --git a/src/fides/api/service/privacy_request/request_runner_service.py b/src/fides/api/service/privacy_request/request_runner_service.py index 350cbd412a..2299827a85 100644 --- a/src/fides/api/service/privacy_request/request_runner_service.py +++ b/src/fides/api/service/privacy_request/request_runner_service.py @@ -696,6 +696,7 @@ def _retrieve_child_results( # pylint: disable=R0911 for key, rows in access_result.items(): address = CollectionAddress.from_string(key) + privacy_request_id = None if address.dataset == fides_connector[0]: if not rows: logger.info("No rows found for result entry {}", key) diff --git a/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy.py b/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy.py index 092044dc37..b3d9bd8aa7 100644 --- a/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy.py +++ b/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union from fides.api.service.strategy import Strategy @@ -9,6 +9,6 @@ class PostProcessorStrategy(Strategy): @abstractmethod def process( - self, data: Any, identity_data: Dict[str, Any] = None + self, data: Any, identity_data: Optional[Dict[str, Any]] = None ) -> Union[List[Dict[str, Any]], Dict[str, Any]]: """Process data from SaaS connector""" diff --git a/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_filter.py b/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_filter.py index e419ddaa52..9420015973 100644 --- a/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_filter.py +++ b/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_filter.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union import pydash from loguru import logger @@ -51,7 +51,7 @@ def __init__(self, configuration: FilterPostProcessorConfiguration): def process( self, data: Union[List[Dict[str, Any]], Dict[str, Any]], - identity_data: Dict[str, Any] = None, + identity_data: Optional[Dict[str, Any]] = None, ) -> Union[List[Dict[str, Any]], Dict[str, Any]]: """ - data: A list or a dict diff --git a/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py b/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py index a257e461ae..ab3fd0c6b6 100644 --- a/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py +++ b/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union import pydash from loguru import logger @@ -41,7 +41,7 @@ def __init__(self, configuration: UnwrapPostProcessorConfiguration): def process( self, data: Union[List[Dict[str, Any]], Dict[str, Any]], - identity_data: Dict[str, Any] = None, + identity_data: Optional[Dict[str, Any]] = None, ) -> Union[List[Dict[str, Any]], Dict[str, Any]]: """ :param data: A list or dict diff --git a/src/fides/api/service/saas_request/override_implementations/oracle_responsys_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/oracle_responsys_request_overrides.py index e9d5e8dd8a..a091fee0d3 100644 --- a/src/fides/api/service/saas_request/override_implementations/oracle_responsys_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/oracle_responsys_request_overrides.py @@ -272,7 +272,8 @@ def oracle_responsys_profile_list_recipients_delete( SaaSRequestParams( method=HTTPMethod.DELETE, path=f"/rest/api/v1.3/lists/{list_id}/members/{responsys_id}", - ) + ), + [404], ) rows_deleted += 1 diff --git a/src/fides/api/service/storage/storage_authenticator_service.py b/src/fides/api/service/storage/storage_authenticator_service.py index 4560633a54..1b663ca890 100644 --- a/src/fides/api/service/storage/storage_authenticator_service.py +++ b/src/fides/api/service/storage/storage_authenticator_service.py @@ -31,7 +31,7 @@ def secrets_are_valid( def _s3_authenticator(secrets: Dict[StorageSecrets, Any]) -> bool: """Authenticates secrets for s3, returns true if secrets are valid""" try: - get_aws_session(AWSAuthMethod.SECRET_KEYS.value, secrets.dict()) # type: ignore + get_aws_session(AWSAuthMethod.SECRET_KEYS.value, secrets.model_dump(mode="json")) # type: ignore return True except ClientError: return False diff --git a/src/fides/api/service/strategy.py b/src/fides/api/service/strategy.py index b2e116231b..b73d6eadbd 100644 --- a/src/fides/api/service/strategy.py +++ b/src/fides/api/service/strategy.py @@ -26,7 +26,7 @@ def _find_strategy_subclass( def _find_all_strategy_subclasses( - cls: Type[T], subs: List[Type[T]] = None + cls: Type[T], subs: Optional[List[Type[T]]] = None ) -> List[Type[T]]: if subs is None: subs = [] diff --git a/src/fides/api/task/create_request_tasks.py b/src/fides/api/task/create_request_tasks.py index 1402d033c1..27f9c3698c 100644 --- a/src/fides/api/task/create_request_tasks.py +++ b/src/fides/api/task/create_request_tasks.py @@ -188,7 +188,8 @@ def base_task_data( # when executing the node, so we don't have to recalculate incoming # and outgoing edges. collection_representation = json.loads( - dataset_graph.nodes[node].collection.json() + # Serialize with duck typing so we get the nested sub fields as well + dataset_graph.nodes[node].collection.model_dump_json(serialize_as_any=True) ) # Saves traversal details based on data dependencies like incoming edges # and input keys, also useful for building the Execution Node diff --git a/src/fides/api/task/execute_request_tasks.py b/src/fides/api/task/execute_request_tasks.py index b849aeb1bf..61e500044a 100644 --- a/src/fides/api/task/execute_request_tasks.py +++ b/src/fides/api/task/execute_request_tasks.py @@ -175,7 +175,7 @@ def queue_downstream_tasks( # Only queue privacy request from the next step if we haven't reached the terminator before. # Multiple pathways could mark the same node as complete, so we may have already reached the # terminator node through a quicker path. - from fides.api.service.privacy_request.request_runner_service import ( + from fides.api.service.privacy_request.request_runner_service import ( # pylint: disable=cyclic-import queue_privacy_request, ) @@ -317,11 +317,12 @@ def run_consent_node( graph_task: GraphTask = create_graph_task( session, request_task, resources ) + access_data: List = [] if upstream_results: # For consent, expected that there is only one upstream node, the root node, # and it holds the identity data (stored in a list for consistency with other # data stored in access_data) - access_data: List = upstream_results[0].get_access_data() or [] + access_data = upstream_results[0].get_access_data() or [] graph_task.consent_request(access_data[0] if access_data else {}) diff --git a/src/fides/api/task/graph_task.py b/src/fides/api/task/graph_task.py index 8ef2c67cf5..8ea09445b8 100644 --- a/src/fides/api/task/graph_task.py +++ b/src/fides/api/task/graph_task.py @@ -35,7 +35,7 @@ ConnectionType, ) from fides.api.models.datasetconfig import DatasetConfig -from fides.api.models.policy import Policy +from fides.api.models.policy import CurrentStep, Policy from fides.api.models.privacy_request import ( ExecutionLog, ExecutionLogStatus, @@ -160,7 +160,12 @@ def result(*args: Any, **kwargs: Any) -> Any: sleep(func_delay) raised_ex = ex self.log_end(action_type, raised_ex) - self.resources.request.cache_failed_checkpoint_details(step=action_type) + # transform ActionType -> CurrentStep type, expected by cache_failed_checkpoint_details + self.resources.request.cache_failed_checkpoint_details( + step=CurrentStep[ + action_type.value + ] # Convert ActionType into a CurrentStep, no longer coerced with Pydantic V2 + ) add_errored_system_status_for_consent_reporting( self.resources.session, self.resources.request, diff --git a/src/fides/api/tasks/__init__.py b/src/fides/api/tasks/__init__.py index 7a7f1fef9c..621c2ea0d2 100644 --- a/src/fides/api/tasks/__init__.py +++ b/src/fides/api/tasks/__init__.py @@ -34,7 +34,7 @@ def get_new_session(self) -> ContextManager[Session]: # only one engine will be instantiated in a given task scope, i.e # once per celery process. if self._task_engine is None: - _task_engine = get_db_engine( + self._task_engine = get_db_engine( config=CONFIG, pool_size=CONFIG.database.task_engine_pool_size, max_overflow=CONFIG.database.task_engine_max_overflow, @@ -42,7 +42,7 @@ def get_new_session(self) -> ContextManager[Session]: # same for the sessionmaker if self._sessionmaker is None: - self._sessionmaker = get_db_session(config=CONFIG, engine=_task_engine) + self._sessionmaker = get_db_session(config=CONFIG, engine=self._task_engine) # but a new session is instantiated each time the method is invoked # to prevent session overlap when requests are executing concurrently diff --git a/src/fides/api/util/connection_type.py b/src/fides/api/util/connection_type.py index ecd273360e..8374429718 100644 --- a/src/fides/api/util/connection_type.py +++ b/src/fides/api/util/connection_type.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Set +from typing import Any, Dict, Set from fides.api.common_exceptions import NoSuchConnectionTypeSecretSchemaError from fides.api.models.connectionconfig import ConnectionType @@ -26,6 +26,115 @@ from fides.api.util.saas_util import load_config_from_string +def transform_v2_to_v1_in_place(schema: Dict[str, Any]) -> None: + """Transform connection secrets from Pydantic V2 format to V1 format for backwards compatibility + since Connection secrets UI is built off of this data. + + This is error prone and is subject to change as we add more types of schemas. + We should consider a refactor of both the frontend and backend here. + """ + + def swap_defs_with_definitions(defn: str) -> str: + """Reverting to v1 schema format for definitions""" + return defn.replace("#/$defs", "#/definitions") + + def transform_any_of(field_attributes_mapping: Dict[str, Any]) -> None: + for attributes in field_attributes_mapping.values(): + # Transforming Pydantic V2 schemas -> Pydantic V1 schemas + if attributes.get("anyOf"): + anyOf = attributes.get("anyOf") + for type_annotation in anyOf: + # If field has a null default, V2 returns multiple types in a list + # of anyOf which isn't compatible with current UI. Returning + # the first non-null type instead. + # + # BEFORE + # "test_email_address": { + # "anyOf": [ + # { + # "format": "email", + # "type": "string" + # }, + # { + # "type": "null" + # } + # ] + # } + # AFTER + # "test_email_address": { + # "type": "string", + # "format": "email" + # }, + if "type" in type_annotation and type_annotation["type"] != "null": + for key, val in type_annotation.items(): + attributes[key] = val + break + # Nested schemas have more complex formatting, they reference a + # more detailed definition elsewhere in the schema + # Advanced settings don't appear to be connected in the UI. + # BEFORE + # "advanced_settings": { + # "anyOf": [ + # { + # "$ref": "#/$defs/AdvancedSettings" + # }, + # { + # "type": "null" + # } + # ] + # } + # AFTER + # "advanced_settings": { + # "title": "Advanced Settings", + # "default": { + # "identity_types": { + # "email": true, + # "phone_number": false + # } + # }, + # "allOf": [ + # { + # "$ref": "#/definitions/AdvancedSettings" + # } + # ] + # } + if "$ref" in type_annotation: + attributes["allOf"] = [ + { + "$ref": swap_defs_with_definitions( + type_annotation["$ref"] + ) + } + ] + break + + attributes.pop("anyOf") + + if "default" in attributes and attributes["default"] is None: + # Backwards compatible with UI, this affects what fields + # show up as required + attributes.pop("default") + + if attributes.get("$ref"): + # V1 called it "#/$defs", V2 dalls it "#/definitions/" + attributes["$ref"] = swap_defs_with_definitions(attributes["$ref"]) + + transform_any_of(schema["properties"]) + + for attributes in schema["properties"].values(): + if attributes.get("allOf"): + for defs in attributes.get("allOf"): + for key, val in defs.items(): + defs[key] = swap_defs_with_definitions(val) + + if schema.get("$defs"): + schema["definitions"] = schema.get("$defs") + schema.pop("$defs") + for key, definition in schema["definitions"].items(): + if definition.get("properties"): + transform_any_of(definition["properties"]) + + def get_connection_type_secret_schema(*, connection_type: str) -> dict[str, Any]: """Returns the secret fields that should be supplied to authenticate with a particular connection type. @@ -39,7 +148,9 @@ def get_connection_type_secret_schema(*, connection_type: str) -> dict[str, Any] ) if connection_type in [db_type.value for db_type in ConnectionType]: - return secrets_schemas[connection_type].schema() + schema = secrets_schemas[connection_type].schema() + transform_v2_to_v1_in_place(schema) + return schema connector_template = ConnectorRegistry.get_connector_template(connection_type) if not connector_template: @@ -49,6 +160,7 @@ def get_connection_type_secret_schema(*, connection_type: str) -> dict[str, Any] config = SaaSConfig(**load_config_from_string(connector_template.config)) schema = SaaSSchemaFactory(config).get_saas_schema().schema() + transform_v2_to_v1_in_place(schema) # rearrange the default order of the properties generated by Pydantic # to reflect the order defined in the connector_params and external_references @@ -57,7 +169,7 @@ def get_connection_type_secret_schema(*, connection_type: str) -> dict[str, Any] order.extend( [ external_reference.name - for external_reference in config.external_references + for external_reference in config.external_references # pylint:disable=not-an-iterable ] ) @@ -67,6 +179,8 @@ def get_connection_type_secret_schema(*, connection_type: str) -> dict[str, Any] schema["properties"].items(), key=lambda item: order.index(item[0]) ) } + if schema.get("additionalProperties"): + schema.pop("additionalProperties") return schema diff --git a/src/fides/api/util/connection_util.py b/src/fides/api/util/connection_util.py index d4b8b9da90..f88ebfdda9 100644 --- a/src/fides/api/util/connection_util.py +++ b/src/fides/api/util/connection_util.py @@ -1,10 +1,10 @@ -from typing import List, Optional +from typing import Annotated, List, Optional from fastapi import Depends, HTTPException +from fastapi.encoders import jsonable_encoder from fideslang.validation import FidesKey from loguru import logger -from pydantic import ValidationError -from pydantic.types import conlist +from pydantic import Field, ValidationError from sqlalchemy.orm import Session from starlette.status import ( HTTP_400_BAD_REQUEST, @@ -113,10 +113,19 @@ def validate_secrets( "Validating secrets on connection config with key '{}'", connection_config.key, ) - connection_secrets = schema.parse_obj(request_body) + connection_secrets = schema.model_validate(request_body) except ValidationError as e: + # Intentionally excluding the pydantic url, input, and ctx fields from the error response to hide + # potentially sensitive information + errors = e.errors(include_url=False, include_input=False) + for err in errors: + # Additionally, manually remove the context from the error message - + # this may contain sensitive information + err.pop("ctx", None) + raise HTTPException( - status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.errors() + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=jsonable_encoder(errors), ) # SaaS secrets with external references must go through extra validation @@ -134,7 +143,7 @@ def validate_secrets( def patch_connection_configs( db: Session, - configs: conlist(CreateConnectionConfigurationWithSecrets, max_items=50), # type: ignore + configs: Annotated[List[CreateConnectionConfigurationWithSecrets], Field(max_length=50)], # type: ignore system: Optional[System] = None, ) -> BulkPutConnectionConfiguration: created_or_updated: List[ConnectionConfigurationResponse] = [] @@ -143,9 +152,11 @@ def patch_connection_configs( for config in configs: # Retrieve the existing connection config from the database - existing_connection_config = ConnectionConfig.get_by( - db, field="key", value=config.key - ) + existing_connection_config = None + if config.key: + existing_connection_config = ConnectionConfig.get_by( + db, field="key", value=config.key + ) if config.connection_type == "saas": if config.secrets: @@ -201,18 +212,32 @@ def patch_connection_configs( status_code=HTTP_400_BAD_REQUEST, detail=exc.args[0], ) + except ValidationError as e: + # The "input" potentially contains sensitive info and the Pydantic-specific "url" is not helpful + errors = e.errors(include_url=False, include_input=False) + for err in errors: + # Additionally, manually remove the context from the error message - + # this may contain sensitive information + err.pop("ctx", None) + + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=jsonable_encoder(errors), + ) connection_config.secrets = validate_secrets( - db, template_values.secrets, connection_config - ).dict() + db, + template_values.secrets, + connection_config, + ).model_dump(mode="json") connection_config.save(db=db) created_or_updated.append( ConnectionConfigurationResponse(**connection_config.__dict__) ) continue - orig_data = config.dict().copy() - config_dict = config.dict() + orig_data = config.model_dump(serialize_as_any=True, mode="json").copy() + config_dict = config.model_dump(serialize_as_any=True, exclude_unset=True) config_dict.pop("saas_connector_type", None) if existing_connection_config: @@ -220,7 +245,7 @@ def patch_connection_configs( key: value for key, value in { **existing_connection_config.__dict__, - **config.dict(), + **config.model_dump(serialize_as_any=True, exclude_unset=True), }.items() if isinstance(value, bool) or value } diff --git a/src/fides/api/util/cors_middleware_utils.py b/src/fides/api/util/cors_middleware_utils.py index 242efcf6ab..18fba79843 100644 --- a/src/fides/api/util/cors_middleware_utils.py +++ b/src/fides/api/util/cors_middleware_utils.py @@ -4,26 +4,40 @@ from fastapi.middleware import Middleware from fastapi.middleware.cors import CORSMiddleware +from fides.api.custom_types import URLOriginString + def update_cors_middleware( app: FastAPI, - allow_origins: Iterable[str], + allow_origins: Iterable[URLOriginString], allow_origin_regex: Optional[Pattern[Any]], ) -> None: """ Update the CORSMiddleware of the provided app with the provided origin parameters. + + In order to update CORSMiddleware after the app has already started, reversing the changes made here: + https://github.com/encode/starlette/pull/2017/files """ existing_middleware = find_cors_middleware(app) + if existing_middleware: app.user_middleware.remove(existing_middleware) - app.add_middleware( - CORSMiddleware, - allow_origins=allow_origins, - allow_origin_regex=allow_origin_regex, - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], + + app.user_middleware.insert( + 0, + Middleware( + CORSMiddleware, + allow_origins=allow_origins, + allow_origin_regex=allow_origin_regex, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ), ) + # In more recent starlette versions, you cannot add middleware after an application has started. + # We have an endpoint that lets us update cors origins. I'm largely attempting to restore the + # behavior starlette was originally providing here. + app.middleware_stack = app.build_middleware_stack() def find_cors_middleware(app: FastAPI) -> Optional[Middleware]: diff --git a/src/fides/api/util/data_category.py b/src/fides/api/util/data_category.py index f0608d6edc..fa8f5b4414 100644 --- a/src/fides/api/util/data_category.py +++ b/src/fides/api/util/data_category.py @@ -16,7 +16,10 @@ def generate_fides_data_categories() -> Type[EnumType]: """Programmatically generated the DataCategory enum based on the imported Fides data.""" FidesDataCategory = EnumType( # type: ignore "FidesDataCategory", - {cat.fides_key: cat.fides_key for cat in DEFAULT_TAXONOMY.data_category}, + { + cat.fides_key: cat.fides_key + for cat in DEFAULT_TAXONOMY.data_category # pylint:disable=not-an-iterable + }, # pylint:disable=not-an-iterable ) return FidesDataCategory diff --git a/src/fides/api/util/errors.py b/src/fides/api/util/errors.py index 2142683727..3a82ebfebc 100644 --- a/src/fides/api/util/errors.py +++ b/src/fides/api/util/errors.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Optional + from fastapi import HTTPException, status @@ -96,7 +98,7 @@ def __init__( resource_type: str, fides_key: str, action: str = "modify", - error_message: str = None, + error_message: Optional[str] = None, ) -> None: error = ( error_message or f"cannot {action} a resource where 'is_default' is true" diff --git a/src/fides/api/util/storage_util.py b/src/fides/api/util/storage_util.py index 13d9013ca0..8dc36d0b4e 100644 --- a/src/fides/api/util/storage_util.py +++ b/src/fides/api/util/storage_util.py @@ -38,7 +38,7 @@ def get_schema_for_secrets( ) try: - return schema.parse_obj(secrets) # type: ignore + return schema.model_validate(secrets) # type: ignore except ValidationError as exc: # Pydantic requires validators raise either a ValueError, TypeError, or AssertionError # so this exception is cast into a `ValueError`. diff --git a/src/fides/cli/__init__.py b/src/fides/cli/__init__.py index 30bf127d81..f244952d0f 100644 --- a/src/fides/cli/__init__.py +++ b/src/fides/cli/__init__.py @@ -41,7 +41,7 @@ from .commands.view import view from .exceptions import LocalModeException -CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) +CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} LOCAL_COMMANDS = [deploy, evaluate, generate, init, scan, parse, view, webserver] LOCAL_COMMAND_NAMES = {command.name for command in LOCAL_COMMANDS} API_COMMANDS = [ diff --git a/src/fides/cli/commands/ungrouped.py b/src/fides/cli/commands/ungrouped.py index 160288c247..ea07c32005 100644 --- a/src/fides/cli/commands/ungrouped.py +++ b/src/fides/cli/commands/ungrouped.py @@ -307,7 +307,7 @@ def parse(ctx: click.Context, manifests_dir: str, verbose: bool = False) -> None """ taxonomy = _parse.parse(manifests_dir=manifests_dir) if verbose: - pretty_echo(taxonomy.dict(), color="green") + pretty_echo(taxonomy.model_dump(mode="json"), color="green") @click.command() # type: ignore diff --git a/src/fides/cli/commands/view.py b/src/fides/cli/commands/view.py index 2a9e965e63..399c7f5265 100644 --- a/src/fides/cli/commands/view.py +++ b/src/fides/cli/commands/view.py @@ -34,7 +34,7 @@ def view_config( _Note: To see the configuration values being used by the webserver, `GET` the `/api/v1/config` endpoint._ """ config = ctx.obj["CONFIG"] - config_dict = config.dict(exclude_unset=exclude_unset) + config_dict = config.model_dump(exclude_unset=exclude_unset) if section: config_dict = config_dict[section] @@ -57,4 +57,4 @@ def view_credentials(ctx: click.Context) -> None: raise SystemExit(1) print_divider() - print(dumps(credentials.dict())) + print(dumps(credentials.model_dump(mode="json"))) diff --git a/src/fides/common/utils.py b/src/fides/common/utils.py index c0bfc6c242..216a4447b6 100644 --- a/src/fides/common/utils.py +++ b/src/fides/common/utils.py @@ -74,5 +74,5 @@ def check_response(response: requests.Response) -> requests.Response: logger.error(response.status_code) logger.error(response.text) raise json_error - else: - return response + + return response diff --git a/src/fides/config/__init__.py b/src/fides/config/__init__.py index 4a14d6d767..72e1ab5f1e 100644 --- a/src/fides/config/__init__.py +++ b/src/fides/config/__init__.py @@ -5,13 +5,16 @@ from functools import lru_cache from os import getenv -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Type, Union import toml from loguru import logger as log -from pydantic import Field -from pydantic.class_validators import _FUNCS -from pydantic.env_settings import SettingsSourceCallable +from pydantic import ConfigDict, Field +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, +) from fides.common.utils import echo_red @@ -82,18 +85,19 @@ class FidesConfig(FidesSettings): security: SecuritySettings user: UserSettings - class Config: # pylint: disable=C0115 - case_sensitive = True + model_config = SettingsConfigDict(case_sensitive=True) - @classmethod - def customise_sources( - cls, - init_settings: SettingsSourceCallable, - env_settings: SettingsSourceCallable, - file_secret_settings: SettingsSourceCallable, - ) -> Tuple[SettingsSourceCallable, ...]: - """Set environment variables to take precedence over init values.""" - return env_settings, init_settings, file_secret_settings + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + """Set environment variables to take precedence over init values.""" + return env_settings, init_settings, file_secret_settings def log_all_config_values(self) -> None: """Output DEBUG logs of all the config values.""" @@ -108,7 +112,7 @@ def log_all_config_values(self) -> None: self.execution, self.admin_ui, ]: - for key, value in settings.dict().items(): # type: ignore + for key, value in settings.model_dump(mode="json").items(): # type: ignore log.debug( f"Using config: {settings.Config.env_prefix}{key.upper()} = {value}", # type: ignore ) @@ -120,7 +124,7 @@ def censor_config(config: Union[FidesConfig, Dict[str, Any]]) -> Dict[str, Any]: strip out any keys not specified in the `CONFIG_KEY_ALLOWLIST` above. """ if not isinstance(config, Dict): - as_dict = config.dict() + as_dict = config.model_dump(mode="json") else: as_dict = config filtered: Dict[str, Any] = {} @@ -163,7 +167,7 @@ def build_config(config_dict: Dict[str, Any]) -> FidesConfig: } for key, value in settings_map.items(): - settings_map[key] = value.parse_obj(config_dict.get(key, {})) + settings_map[key] = value.model_validate(config_dict.get(key, {})) # Logic for populating the user-defined credentials sub-settings. # this is done to allow overrides without typed pydantic models @@ -190,10 +194,6 @@ def get_config(config_path_override: str = "", verbose: bool = False) -> FidesCo This will fail if the first encountered configuration file is invalid. """ - # This prevents a Pydantic validator reuse error. For context see - # https://github.com/streamlit/streamlit/issues/3218 - _FUNCS.clear() - env_config_path = getenv(DEFAULT_CONFIG_PATH_ENV_VAR) config_path = config_path_override or env_config_path or DEFAULT_CONFIG_PATH diff --git a/src/fides/config/admin_ui_settings.py b/src/fides/config/admin_ui_settings.py index 569f9547eb..67cacba456 100644 --- a/src/fides/config/admin_ui_settings.py +++ b/src/fides/config/admin_ui_settings.py @@ -1,6 +1,9 @@ from typing import Optional -from pydantic import AnyHttpUrl, Field +from pydantic import Field, SerializeAsAny +from pydantic_settings import SettingsConfigDict + +from fides.api.custom_types import AnyHttpUrlStringRemovesSlash from .fides_settings import FidesSettings @@ -11,9 +14,7 @@ class AdminUISettings(FidesSettings): enabled: bool = Field( default=True, description="Toggle whether the Admin UI is served." ) - url: Optional[AnyHttpUrl] = Field( + url: SerializeAsAny[Optional[AnyHttpUrlStringRemovesSlash]] = Field( default=None, description="The base URL for the Admin UI." ) - - class Config: - env_prefix = "FIDES__ADMIN_UI__" + model_config = SettingsConfigDict(env_prefix="FIDES__ADMIN_UI__") diff --git a/src/fides/config/celery_settings.py b/src/fides/config/celery_settings.py index 64ecf1349d..6987236c10 100644 --- a/src/fides/config/celery_settings.py +++ b/src/fides/config/celery_settings.py @@ -1,4 +1,5 @@ from pydantic import Field +from pydantic_settings import SettingsConfigDict from .fides_settings import FidesSettings @@ -21,6 +22,4 @@ class CelerySettings(FidesSettings): description="If true, tasks are executed locally instead of being sent to the queue. " "If False, tasks are sent to the queue.", ) - - class Config: - env_prefix = ENV_PREFIX + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX) diff --git a/src/fides/config/cli_settings.py b/src/fides/config/cli_settings.py index 92f3171b58..97906d4b50 100644 --- a/src/fides/config/cli_settings.py +++ b/src/fides/config/cli_settings.py @@ -1,13 +1,16 @@ """This module defines the settings for everything related to the CLI.""" -from typing import Dict, Optional +from typing import Optional +from fideslang.validation import AnyHttpUrlString from fideslog.sdk.python.utils import FIDESCTL_CLI, generate_client_id -from pydantic import AnyHttpUrl, Field, validator +from pydantic import Field, SerializeAsAny, ValidationInfo, field_validator +from pydantic_settings import SettingsConfigDict -from .fides_settings import FidesSettings +from .fides_settings import FidesSettings, port_integer_converter # pylint: disable=C0115,C0116, E0213 + ENV_PREFIX = "FIDES__CLI__" @@ -17,6 +20,7 @@ class CLISettings(FidesSettings): analytics_id: str = Field( default=generate_client_id(FIDESCTL_CLI), description="A fully anonymized unique identifier that is automatically generated by the application. Used for anonymous analytics when opted-in.", + validate_default=True, ) local_mode: bool = Field( default=False, @@ -31,19 +35,20 @@ class CLISettings(FidesSettings): server_port: str = Field( default="8080", description="The port of the Fides webserver" ) - server_url: Optional[AnyHttpUrl] = Field( + server_url: SerializeAsAny[Optional[AnyHttpUrlString]] = Field( default=None, description="The full server url generated from the other server configuration values.", exclude=True, + validate_default=True, ) - @validator("server_url", always=True) + @field_validator("server_url") @classmethod - def get_server_url(cls, value: str, values: Dict) -> str: - "Create the server_url." - host = values["server_host"] - port = int(values["server_port"]) - protocol = values["server_protocol"] + def get_server_url(cls, value: str, info: ValidationInfo) -> str: + """Create the server_url.""" + host = info.data.get("server_host") + port: int = port_integer_converter(info, "server_port") + protocol = info.data.get("server_protocol") server_url = "{}://{}{}".format( protocol, @@ -53,12 +58,11 @@ def get_server_url(cls, value: str, values: Dict) -> str: return server_url - @validator("analytics_id", always=True) + @field_validator("analytics_id") def ensure_not_empty(cls, value: str) -> str: """ Validate that the `analytics_id` is not `""`. """ return value if value != "" else generate_client_id(FIDESCTL_CLI) - class Config: - env_prefix = ENV_PREFIX + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX, coerce_numbers_to_str=True) diff --git a/src/fides/config/config_proxy.py b/src/fides/config/config_proxy.py index 22c20eecc2..20a8d69daf 100644 --- a/src/fides/config/config_proxy.py +++ b/src/fides/config/config_proxy.py @@ -3,9 +3,10 @@ from typing import Any, Callable, Iterable, List, Optional, Set from fastapi.applications import FastAPI -from pydantic import AnyUrl +from pydantic import SerializeAsAny from sqlalchemy.orm import Session +from fides.api.custom_types import AnyHttpUrlStringRemovesSlash, URLOriginString from fides.api.models.application_config import ApplicationConfig from fides.api.schemas.storage.storage import StorageType from fides.api.util.cors_middleware_utils import update_cors_middleware @@ -86,7 +87,7 @@ class AdminUISettingsProxy(ConfigProxyBase): prefix = "admin_ui" enabled: bool - url: Optional[str] + url: SerializeAsAny[Optional[AnyHttpUrlStringRemovesSlash]] = None class NotificationSettingsProxy(ConfigProxyBase): @@ -139,7 +140,7 @@ class SecuritySettingsProxy(ConfigProxyBase): # for advanced usage of non-URLs, e.g. wildcards (`*`), the related # `cors_origin_regex` property should be used. # this is explicitly _not_ accessible via API - it must be used with care. - cors_origins: List[AnyUrl] + cors_origins: SerializeAsAny[List[URLOriginString]] class ConsentSettingsProxy(ConfigProxyBase): diff --git a/src/fides/config/consent_settings.py b/src/fides/config/consent_settings.py index a16971b20e..e9fa98bb87 100644 --- a/src/fides/config/consent_settings.py +++ b/src/fides/config/consent_settings.py @@ -1,6 +1,5 @@ -from typing import Any, Dict - -from pydantic import Field, root_validator +from pydantic import Field, model_validator +from pydantic_settings import SettingsConfigDict from .fides_settings import FidesSettings @@ -19,16 +18,14 @@ class ConsentSettings(FidesSettings): default=False, description="Whether or not vendor purposes can be globally overridden.", ) + model_config = SettingsConfigDict(env_prefix="FIDES__CONSENT__") - class Config: - env_prefix = "FIDES__CONSENT__" - - @root_validator - def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @model_validator(mode="after") + def validate_fields(self) -> "ConsentSettings": """AC mode only works if TCF mode is also enabled""" - tcf_mode = values.get("tcf_enabled") - ac_mode = values.get("ac_enabled") - override_vendor_purposes = values.get("override_vendor_purposes") + tcf_mode = self.tcf_enabled + ac_mode = self.ac_enabled + override_vendor_purposes = self.override_vendor_purposes if ac_mode and not tcf_mode: raise ValueError("AC cannot be enabled unless TCF mode is also enabled.") @@ -38,4 +35,4 @@ def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: "Override vendor purposes cannot be true unless TCF mode is also enabled." ) - return values + return self diff --git a/src/fides/config/create.py b/src/fides/config/create.py index 2df6d0cd1d..48378b00bc 100644 --- a/src/fides/config/create.py +++ b/src/fides/config/create.py @@ -8,7 +8,7 @@ import toml from click import echo -from pydantic import BaseSettings +from pydantic_settings import BaseSettings from fides.cli.utils import request_analytics_consent from fides.config import FidesConfig, build_config @@ -32,8 +32,10 @@ def get_nested_settings(config: FidesConfig) -> Dict[str, BaseSettings]: """ nested_settings = { settings_name - for settings_name, settings_info in config.schema()["properties"].items() - if not settings_info.get("type") + for settings_name, settings_info in config.model_json_schema()[ + "properties" + ].items() + if not settings_info.get("type") and not settings_info.get("anyOf") } nested_settings_objects = { @@ -54,15 +56,27 @@ def format_value_for_toml(value: str, value_type: str) -> str: return value -def build_field_documentation( - field_name: str, field_info: Dict[str, str] -) -> Optional[str]: - """Build a docstring for an individual docstring.""" +def build_field_documentation(field_name: str, field_info: Dict) -> Optional[str]: + """Build a docstring for an individual docstring. + + This is error prone - this is attempts to pull data out of the Pydantic schema, but + not every case is handled. + """ try: - field_type = field_info["type"] - field_description = "\n".join( + # Singular field types under "type" + field_type: str = field_info.get("type") or "" + if not field_type: + # Union field types are under "anyOf" + any_of: List[Dict[str, str]] = field_info.get("anyOf") or [] + for type_annotation in any_of: + if type_annotation["type"] != "null": + # Getting first non-null + field_type = type_annotation["type"] + break + + field_description: str = "\n".join( wrap( - text=field_info["description"], + text=field_info.get("description") or "", width=71, subsequent_indent="# ", initial_indent="# ", @@ -114,7 +128,7 @@ def convert_settings_to_toml_docs(settings_name: str, settings: BaseSettings) -> The string is expected to be valid TOML. """ settings_schema = settings.schema() - included_keys = set(settings.dict().keys()) + included_keys = set(settings.model_dump(mode="json").keys()) title_header = build_section_header(settings_name) # Build the Section docstring diff --git a/src/fides/config/credentials_settings.py b/src/fides/config/credentials_settings.py index 4b0c35b833..21eb8e4314 100644 --- a/src/fides/config/credentials_settings.py +++ b/src/fides/config/credentials_settings.py @@ -68,7 +68,7 @@ def get_config_database_credentials( ) -> Optional[DatabaseConfig]: credentials_dict = credentials_config.get(credentials_id, None) parsed_config = ( - DatabaseConfig.parse_obj(credentials_dict) if credentials_dict else None + DatabaseConfig.model_validate(credentials_dict) if credentials_dict else None ) return parsed_config @@ -77,7 +77,9 @@ def get_config_okta_credentials( credentials_config: Dict[str, Dict], credentials_id: str ) -> Optional[OktaConfig]: credentials_dict = credentials_config.get(credentials_id, None) - parsed_config = OktaConfig.parse_obj(credentials_dict) if credentials_dict else None + parsed_config = ( + OktaConfig.model_validate(credentials_dict) if credentials_dict else None + ) return parsed_config @@ -85,7 +87,9 @@ def get_config_aws_credentials( credentials_config: Dict[str, Dict], credentials_id: str ) -> Optional[AWSConfig]: credentials_dict = credentials_config.get(credentials_id, None) - parsed_config = AWSConfig.parse_obj(credentials_dict) if credentials_dict else None + parsed_config = ( + AWSConfig.model_validate(credentials_dict) if credentials_dict else None + ) return parsed_config @@ -94,7 +98,7 @@ def get_config_bigquery_credentials( ) -> Optional[BigQueryConfig]: credentials_dict = credentials_config.get(credentials_id) parsed_credentials = ( - KeyfileCreds.parse_obj(credentials_dict) if credentials_dict else None + KeyfileCreds.model_validate(credentials_dict) if credentials_dict else None ) parsed_config = ( BigQueryConfig(dataset=dataset, keyfile_creds=parsed_credentials) diff --git a/src/fides/config/database_settings.py b/src/fides/config/database_settings.py index 5ad9d938ef..9698050cb1 100644 --- a/src/fides/config/database_settings.py +++ b/src/fides/config/database_settings.py @@ -3,14 +3,15 @@ # pylint: disable=C0115,C0116, E0213 from copy import deepcopy -from typing import Dict, Optional, Union, cast +from typing import Dict, Optional, cast from urllib.parse import quote, quote_plus, urlencode -from pydantic import Field, PostgresDsn, validator +from pydantic import Field, PostgresDsn, ValidationInfo, field_validator +from pydantic_settings import SettingsConfigDict from fides.config.utils import get_test_mode -from .fides_settings import FidesSettings +from .fides_settings import FidesSettings, port_integer_converter ENV_PREFIX = "FIDES__DATABASE__" @@ -96,51 +97,57 @@ class DatabaseSettings(FidesSettings): exclude=True, ) - @validator("password", pre=True) + @field_validator("password", mode="before") + @classmethod def escape_password(cls, value: Optional[str]) -> Optional[str]: """Escape password""" if value and isinstance(value, str): return quote_plus(value) return value - @validator("sync_database_uri", pre=True) + @field_validator("sync_database_uri", mode="before") @classmethod def assemble_sync_database_uri( - cls, value: Optional[str], values: Dict[str, Union[str, Dict]] + cls, value: Optional[str], info: ValidationInfo ) -> str: """Join DB connection credentials into a connection string""" if isinstance(value, str) and value: return value - db_name = values["test_db"] if get_test_mode() else values["db"] + port: int = port_integer_converter(info) + db_name = info.data.get("test_db") if get_test_mode() else info.data.get("db") return str( - PostgresDsn.build( + PostgresDsn.build( # pylint: disable=no-member scheme="postgresql+psycopg2", - user=values["user"], - password=values["password"], - host=values["server"], - port=values.get("port"), - path=f"/{db_name or ''}", - query=urlencode( - cast(Dict, values["params"]), quote_via=quote, safe="/" + username=info.data.get("user"), + password=info.data.get("password"), + host=info.data.get("server"), + port=port, + path=f"{db_name or ''}", + query=( + urlencode( + cast(Dict, info.data.get("params")), quote_via=quote, safe="/" + ) + if info.data.get("params") + else None ), ) ) - @validator("async_database_uri", pre=True) + @field_validator("async_database_uri", mode="before") @classmethod def assemble_async_database_uri( - cls, value: Optional[str], values: Dict[str, Union[str, Dict]] + cls, value: Optional[str], info: ValidationInfo ) -> str: """Join DB connection credentials into an async connection string.""" if isinstance(value, str) and value: return value - db_name = values["test_db"] if get_test_mode() else values["db"] + db_name = info.data.get("test_db") if get_test_mode() else info.data.get("db") # Workaround https://github.com/MagicStack/asyncpg/issues/737 # Required due to the unique way in which Asyncpg handles SSL - params = cast(Dict, deepcopy(values["params"])) + params = cast(Dict, deepcopy(info.data.get("params"))) if "sslmode" in params: params["ssl"] = params.pop("sslmode") # This must be constructed in fides.api.db.session as part of the ssl context @@ -148,61 +155,69 @@ def assemble_async_database_uri( params.pop("sslrootcert", None) # End workaround + port: int = port_integer_converter(info) return str( - PostgresDsn.build( + PostgresDsn.build( # pylint: disable=no-member scheme="postgresql+asyncpg", - user=values["user"], - password=values["password"], - host=values["server"], - port=values.get("port"), - path=f"/{db_name or ''}", - query=urlencode(params, quote_via=quote, safe="/"), + username=info.data.get("user"), + password=info.data.get("password"), + host=info.data.get("server"), + port=port, + path=f"{db_name or ''}", + query=urlencode(params, quote_via=quote, safe="/") if params else None, ) ) - @validator("sqlalchemy_database_uri", pre=True) + @field_validator("sqlalchemy_database_uri", mode="before") @classmethod - def assemble_db_connection( - cls, v: Optional[str], values: Dict[str, Union[str, Dict]] - ) -> str: + def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> str: """Join DB connection credentials into a synchronous connection string.""" if isinstance(v, str) and v: return v + + port: int = port_integer_converter(info) return str( - PostgresDsn.build( + PostgresDsn.build( # pylint: disable=no-member scheme="postgresql", - user=values["user"], - password=values["password"], - host=values["server"], - port=values.get("port"), - path=f"/{values.get('db') or ''}", - query=urlencode( - cast(Dict, values["params"]), quote_via=quote, safe="/" + username=info.data.get("user"), + password=info.data.get("password"), + host=info.data.get("server"), + port=port, + path=f"{info.data.get('db') or ''}", + query=( + urlencode( + cast(Dict, info.data.get("params")), quote_via=quote, safe="/" + ) + if info.data.get("params") + else None ), ) ) - @validator("sqlalchemy_test_database_uri", pre=True) + @field_validator("sqlalchemy_test_database_uri", mode="before") @classmethod - def assemble_test_db_connection( - cls, v: Optional[str], values: Dict[str, Union[str, Dict]] - ) -> str: + def assemble_test_db_connection(cls, v: Optional[str], info: ValidationInfo) -> str: """Join DB connection credentials into a connection string""" if isinstance(v, str) and v: return v + + port: int = port_integer_converter(info) return str( - PostgresDsn.build( + PostgresDsn.build( # pylint: disable=no-member scheme="postgresql", - user=values["user"], - password=values["password"], - host=values["server"], - port=values["port"], - path=f"/{values.get('test_db') or ''}", - query=urlencode( - cast(Dict, values["params"]), quote_via=quote, safe="/" + username=info.data.get("user"), + password=info.data.get("password"), + host=info.data.get("server"), + port=port, + path=f"{info.data.get('test_db') or ''}", + query=( + urlencode( + cast(Dict, info.data.get("params")), quote_via=quote, safe="/" + ) + if info.data.get("params") + else None ), ) ) - class Config: - env_prefix = ENV_PREFIX + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX, coerce_numbers_to_str=True) diff --git a/src/fides/config/execution_settings.py b/src/fides/config/execution_settings.py index 7bcad29b08..3e544c85d9 100644 --- a/src/fides/config/execution_settings.py +++ b/src/fides/config/execution_settings.py @@ -1,6 +1,7 @@ from typing import Optional from pydantic import Field +from pydantic_settings import SettingsConfigDict from .fides_settings import FidesSettings @@ -60,6 +61,4 @@ class ExecutionSettings(FidesSettings): default=False, description="Temporary flag to switch to using DSR 3.0 to process your tasks.", ) - - class Config: - env_prefix = ENV_PREFIX + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX) diff --git a/src/fides/config/fides_settings.py b/src/fides/config/fides_settings.py index d2e21b5437..c379337fbb 100644 --- a/src/fides/config/fides_settings.py +++ b/src/fides/config/fides_settings.py @@ -2,25 +2,47 @@ # pylint: disable=C0115,C0116, E0213 -from typing import Tuple +from typing import Optional, Tuple, Type -from pydantic import BaseSettings, Extra -from pydantic.env_settings import SettingsSourceCallable +from pydantic import ValidationInfo +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, +) class FidesSettings(BaseSettings): """Class used as a base model for configuration subsections.""" - class Config: - # Need to allow extras because the inheriting class will have more info - extra = Extra.allow - - @classmethod - def customise_sources( - cls, - init_settings: SettingsSourceCallable, - env_settings: SettingsSourceCallable, - file_secret_settings: SettingsSourceCallable, - ) -> Tuple[SettingsSourceCallable, ...]: - """Set environment variables to take precedence over init values.""" - return env_settings, init_settings, file_secret_settings + model_config = SettingsConfigDict(extra="allow") + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + """Set environment variables to take precedence over init values.""" + return env_settings, init_settings, file_secret_settings + + +def port_integer_converter( + info: ValidationInfo, port_name: Optional[str] = None +) -> int: + """ + Convert supplied port value to an integer. + + "Before" field validators mean that port is not yet guaranteed to exist or be the appropriate type + """ + port = info.data.get(port_name or "port") + + if not isinstance(port, (str, int)): # This check also helps satisfy mypy + raise ValueError( + "Port must be supplied and able to be converted to an integer." + ) + + return int(port) diff --git a/src/fides/config/logging_settings.py b/src/fides/config/logging_settings.py index 0c23ab107b..bf673ec490 100644 --- a/src/fides/config/logging_settings.py +++ b/src/fides/config/logging_settings.py @@ -4,7 +4,8 @@ import os from logging import CRITICAL, DEBUG, ERROR, INFO, WARNING, getLevelName -from pydantic import Field, validator +from pydantic import Field, field_validator +from pydantic_settings import SettingsConfigDict from .fides_settings import FidesSettings from .utils import get_dev_mode @@ -37,7 +38,7 @@ class LoggingSettings(FidesSettings): description="If True, PII values will display unmasked in log output. This variable should always be set to 'False' in production systems.", ) - @validator("destination", pre=True) + @field_validator("destination", mode="before") @classmethod def get_destination(cls, value: str) -> str: """ @@ -45,7 +46,7 @@ def get_destination(cls, value: str) -> str: """ return value if os.path.exists(value) else "" - @validator("level", pre=True) + @field_validator("level", mode="before") @classmethod def validate_log_level(cls, value: str) -> str: """Ensure the provided LEVEL is a valid value.""" @@ -69,7 +70,7 @@ def validate_log_level(cls, value: str) -> str: return value - @validator("serialization", pre=True) + @field_validator("serialization", mode="before") @classmethod def get_serialization(cls, value: str) -> str: """ @@ -78,5 +79,4 @@ def get_serialization(cls, value: str) -> str: value = value.lower() return value if value == "json" else "" - class Config: - env_prefix = ENV_PREFIX + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX) diff --git a/src/fides/config/notification_settings.py b/src/fides/config/notification_settings.py index cac4192817..88d61d0a20 100644 --- a/src/fides/config/notification_settings.py +++ b/src/fides/config/notification_settings.py @@ -1,6 +1,7 @@ from typing import Optional -from pydantic import Field, validator +from pydantic import Field, field_validator +from pydantic_settings import SettingsConfigDict from .fides_settings import FidesSettings @@ -31,7 +32,7 @@ class NotificationSettings(FidesSettings): description="When set to True, enables property specific messaging feature, otherwise fall back on the messaging template type env flags set above.", ) - @validator("notification_service_type", pre=True) + @field_validator("notification_service_type", mode="before") @classmethod def validate_notification_service_type(cls, value: Optional[str]) -> Optional[str]: """Ensure the provided type is a valid value.""" @@ -51,5 +52,4 @@ def validate_notification_service_type(cls, value: Optional[str]) -> Optional[st return value - class Config: - env_prefix = ENV_PREFIX + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX) diff --git a/src/fides/config/redis_settings.py b/src/fides/config/redis_settings.py index 35f8e0cca5..c6a191fc58 100644 --- a/src/fides/config/redis_settings.py +++ b/src/fides/config/redis_settings.py @@ -1,7 +1,8 @@ -from typing import Dict, Optional +from typing import Optional from urllib.parse import quote, quote_plus, urlencode -from pydantic import Field, validator +from pydantic import Field, ValidationInfo, field_validator +from pydantic_settings import SettingsConfigDict from .fides_settings import FidesSettings @@ -70,12 +71,12 @@ class RedisSettings(FidesSettings): exclude=True, ) - @validator("connection_url", pre=True) + @field_validator("connection_url", mode="before") @classmethod def assemble_connection_url( cls, v: Optional[str], - values: Dict[str, str], + info: ValidationInfo, ) -> str: """Join Redis connection credentials into a connection string""" if isinstance(v, str): @@ -84,20 +85,20 @@ def assemble_connection_url( connection_protocol = "redis" params_str = "" - use_tls = values.get("ssl", False) + use_tls = info.data.get("ssl", False) # These vars are intentionally fetched with `or ""` as the default to account # for the edge case where `None` is explicitly set in `values` by Pydantic because # it is not overridden by the config file or an env var - user = values.get("user") or "" - password = values.get("password") or "" - db_index = values.get("db_index") or "" + user = info.data.get("user") or "" + password = info.data.get("password") or "" + db_index = info.data.get("db_index") or "" if use_tls: # If using TLS update the connection URL format connection_protocol = "rediss" - cert_reqs = values.get("ssl_cert_reqs", "none") + cert_reqs = info.data.get("ssl_cert_reqs", "none") params = {"ssl_cert_reqs": quote_plus(cert_reqs)} - if ssl_ca_certs := values.get("ssl_ca_certs", ""): + if ssl_ca_certs := info.data.get("ssl_ca_certs", ""): params["ssl_ca_certs"] = quote(ssl_ca_certs, safe="/") params_str = "?" + urlencode(params, quote_via=quote, safe="/") @@ -107,8 +108,7 @@ def assemble_connection_url( if password or user: auth_prefix = f"{quote_plus(user)}:{quote_plus(password)}@" - connection_url = f"{connection_protocol}://{auth_prefix}{values.get('host', '')}:{values.get('port', '')}/{db_index}{params_str}" + connection_url = f"{connection_protocol}://{auth_prefix}{info.data.get('host', '')}:{info.data.get('port', '')}/{db_index}{params_str}" return connection_url - class Config: - env_prefix = ENV_PREFIX + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX) diff --git a/src/fides/config/security_settings.py b/src/fides/config/security_settings.py index 71b68f6503..779ea1f6ff 100644 --- a/src/fides/config/security_settings.py +++ b/src/fides/config/security_settings.py @@ -1,14 +1,15 @@ """This module handles finding and parsing fides configuration files.""" # pylint: disable=C0115,C0116, E0213 -from typing import Dict, List, Optional, Pattern, Tuple, Union +from typing import List, Optional, Pattern, Tuple, Union import validators -from pydantic import Field, validator +from pydantic import Field, SerializeAsAny, ValidationInfo, field_validator +from pydantic_settings import SettingsConfigDict from slowapi.wrappers import parse_many # type: ignore from fides.api.cryptography.cryptographic_util import generate_salt, hash_with_salt -from fides.api.custom_types import URLOrigin +from fides.api.custom_types import URLOriginString from fides.api.oauth.roles import OWNER from fides.common.api.scope_registry import SCOPE_REGISTRY @@ -31,7 +32,7 @@ class SecuritySettings(FidesSettings): app_encryption_key: str = Field( default="", description="The key used to sign Fides API access tokens." ) - cors_origins: List[URLOrigin] = Field( + cors_origins: SerializeAsAny[List[URLOriginString]] = Field( default_factory=list, description="A list of client addresses allowed to communicate with the Fides webserver.", ) @@ -144,10 +145,10 @@ class SecuritySettings(FidesSettings): description="The timeout in seconds for tunnel connection (open_channel timeout)", ) - @validator("app_encryption_key") + @field_validator("app_encryption_key", mode="before") @classmethod def validate_encryption_key_length( - cls, v: Optional[str], values: Dict[str, str] + cls, v: Optional[str], info: ValidationInfo ) -> Optional[str]: """Validate the encryption key is exactly 32 characters""" @@ -155,13 +156,13 @@ def validate_encryption_key_length( if v == "": return v - if v is None or len(v.encode(values.get("encoding", "UTF-8"))) != 32: + if v is None or len(v.encode(info.data.get("encoding", "UTF-8"))) != 32: raise ValueError( "APP_ENCRYPTION_KEY value must be exactly 32 characters long" ) return v - @validator("cors_origins", pre=True) + @field_validator("cors_origins", mode="before") @classmethod def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str]: """Return a list of valid origins for CORS requests""" @@ -183,29 +184,29 @@ def validate(values: List[str]) -> None: return v raise ValueError(v) - @validator("oauth_root_client_secret_hash") + @field_validator("oauth_root_client_secret_hash", mode="before") @classmethod def assemble_root_access_token( - cls, v: Optional[str], values: Dict[str, str] + cls, v: Optional[str], info: ValidationInfo ) -> Optional[Tuple]: """ Sets a hashed value of the root access key. This is hashed as it is not wise to return a plaintext for of the root credential anywhere in the system. """ - value = values.get("oauth_root_client_secret", "") + value = info.data.get("oauth_root_client_secret", "") if not value: return None - encoding = values.get("encoding", "UTF-8") + encoding = info.data.get("encoding", "UTF-8") salt = generate_salt() hashed_client_id = hash_with_salt(value.encode(encoding), salt.encode(encoding)) oauth_root_client_secret_hash = (hashed_client_id, salt.encode(encoding)) # type: ignore return oauth_root_client_secret_hash - @validator("request_rate_limit") + @field_validator("request_rate_limit") @classmethod def validate_request_rate_limit( cls, @@ -226,7 +227,7 @@ def validate_request_rate_limit( raise ValueError(message) return v - @validator("env") + @field_validator("env") @classmethod def validate_env( cls, @@ -238,5 +239,4 @@ def validate_env( raise ValueError(message) return v - class Config: - env_prefix = ENV_PREFIX + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX) diff --git a/src/fides/config/user_settings.py b/src/fides/config/user_settings.py index 7f12ce799d..09c6ff3f73 100644 --- a/src/fides/config/user_settings.py +++ b/src/fides/config/user_settings.py @@ -4,6 +4,7 @@ from typing import Dict from pydantic import Field +from pydantic_settings import SettingsConfigDict from fides.core.utils import create_auth_header, get_auth_header @@ -42,6 +43,4 @@ class UserSettings(FidesSettings): password: str = Field( default="", description="The password used to log into the Fides webserver." ) - - class Config: - env_prefix = ENV_PREFIX + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX) diff --git a/src/fides/config/utils.py b/src/fides/config/utils.py index 6dcc49a013..c31d4cca78 100644 --- a/src/fides/config/utils.py +++ b/src/fides/config/utils.py @@ -66,5 +66,5 @@ def get_dev_mode() -> bool: "active_default_storage_type", ], "consent": ["override_vendor_purposes"], - "admin-ui": ["enabled", "url"], + "admin_ui": ["enabled", "url"], } diff --git a/src/fides/connectors/aws.py b/src/fides/connectors/aws.py index 64fb60a5bc..00f5459ad5 100644 --- a/src/fides/connectors/aws.py +++ b/src/fides/connectors/aws.py @@ -27,7 +27,7 @@ def get_aws_client(service: str, aws_config: Optional[AWSConfig]) -> Any: Creates boto3 client for a given service. A config is optional to allow for environment variable configuration. """ - config_dict = aws_config.dict() if aws_config else {} + config_dict = aws_config.model_dump(mode="json") if aws_config else {} service_client = boto3.client( service, **config_dict, @@ -244,7 +244,7 @@ def create_redshift_systems( fides_key=cluster["ClusterIdentifier"], name=cluster["ClusterIdentifier"], description=f"Fides Generated Description for Redshift Cluster: {cluster['ClusterIdentifier']}", - meta={(pair["Key"], pair["Value"]) for pair in cluster.get("Tags", {})}, + meta={pair["Key"]: pair["Value"] for pair in cluster.get("Tags", {})}, system_type="redshift_cluster", organization_fides_key=organization_key, fidesctl_meta=SystemMetadata( @@ -280,7 +280,7 @@ def create_rds_systems( fides_key=cluster["DBClusterIdentifier"], name=cluster["DBClusterIdentifier"], description=f"Fides Generated Description for RDS Cluster: {cluster['DBClusterIdentifier']}", - meta={(pair["Key"], pair["Value"]) for pair in cluster.get("TagList", {})}, + meta={pair["Key"]: pair["Value"] for pair in cluster.get("TagList", {})}, system_type="rds_cluster", organization_fides_key=organization_key, fidesctl_meta=SystemMetadata( @@ -297,7 +297,7 @@ def create_rds_systems( fides_key=instance["DBInstanceIdentifier"], name=instance["DBInstanceIdentifier"], description=f"Fides Generated Description for RDS Instance: {instance['DBInstanceIdentifier']}", - meta={(pair["Key"], pair["Value"]) for pair in instance.get("TagList", {})}, + meta={pair["Key"]: pair["Value"] for pair in instance.get("TagList", {})}, system_type="rds_instance", organization_fides_key=organization_key, fidesctl_meta=SystemMetadata( @@ -360,7 +360,7 @@ def create_tagging_dynamodb_system( fides_key=table_name, name=table_name, description=f"Fides Generated Description for DynamoDb table: {table_name}", - meta={(pair["Key"], pair["Value"]) for pair in resource.get("Tags", {})}, + meta={pair["Key"]: pair["Value"] for pair in resource.get("Tags", {})}, system_type="dynamodb_table", organization_fides_key=organization_key, fidesctl_meta=SystemMetadata( @@ -368,7 +368,8 @@ def create_tagging_dynamodb_system( ), privacy_declarations=[], ) - return system + return system + return None def create_tagging_s3_system( @@ -387,7 +388,7 @@ def create_tagging_s3_system( fides_key=bucket_name, name=bucket_name, description=f"Fides Generated Description for S3 bucket: {bucket_name}", - meta={(pair["Key"], pair["Value"]) for pair in resource.get("Tags", {})}, + meta={pair["Key"]: pair["Value"] for pair in resource.get("Tags", {})}, system_type="s3_bucket", organization_fides_key=organization_key, fidesctl_meta=SystemMetadata( diff --git a/src/fides/connectors/bigquery.py b/src/fides/connectors/bigquery.py index 7f46373cb0..46a8850008 100644 --- a/src/fides/connectors/bigquery.py +++ b/src/fides/connectors/bigquery.py @@ -14,7 +14,7 @@ def get_bigquery_engine(bigquery_config: BigQueryConfig) -> Engine: dataset = bigquery_config.dataset engine = create_engine( f"bigquery://{project_id}/{dataset}", - credentials_info=bigquery_config.keyfile_creds.dict(), + credentials_info=bigquery_config.keyfile_creds.model_dump(mode="json"), ) validate_bigquery_engine(engine) return engine diff --git a/src/fides/connectors/models.py b/src/fides/connectors/models.py index cc65d97106..e209c6e529 100644 --- a/src/fides/connectors/models.py +++ b/src/fides/connectors/models.py @@ -1,7 +1,7 @@ """Module that adds models for connectors""" # pylint: disable=C0115,C0116, E0213 -from typing import List, Optional +from typing import ClassVar, List, Optional from pydantic import BaseModel @@ -40,7 +40,7 @@ class BigQueryConfig(BaseModel): dataset: Optional[str] = None keyfile_creds: KeyfileCreds - _required_components: List[str] = ["keyfile_creds"] + _required_components: ClassVar[List[str]] = ["keyfile_creds"] class OktaConfig(BaseModel): diff --git a/src/fides/connectors/okta.py b/src/fides/connectors/okta.py index c30ec50590..57dd82a5c8 100644 --- a/src/fides/connectors/okta.py +++ b/src/fides/connectors/okta.py @@ -23,7 +23,7 @@ def get_okta_client(okta_config: Optional[OktaConfig]) -> OktaClient: Enabled raiseException config to facilitate exception handling """ - config_dict = okta_config.dict() if okta_config else {} + config_dict = okta_config.model_dump(mode="json") if okta_config else {} config_dict["raiseException"] = True okta_client = OktaClient(config_dict) return okta_client diff --git a/src/fides/core/annotate_dataset.py b/src/fides/core/annotate_dataset.py index bc2541dc8b..9954b9169d 100644 --- a/src/fides/core/annotate_dataset.py +++ b/src/fides/core/annotate_dataset.py @@ -9,7 +9,7 @@ from fideslang.manifests import ingest_manifests from fideslang.models import Dataset, DatasetCollection, DatasetField from fideslang.parse import parse_dict -from fideslang.validation import FidesKey, FidesValidationError +from fideslang.validation import FidesKey, FidesValidationError, validate_fides_key from fides.common.utils import echo_green from fides.config import FidesConfig @@ -36,7 +36,7 @@ def validate_data_categories( """ for category in categories: - FidesKey.validate(category) + validate_fides_key(category) if category not in valid_categories: raise ValueError @@ -109,7 +109,7 @@ def annotate_dataset( output_dataset = [] datasets = [ - Dataset.parse_obj(dataset) + Dataset.model_validate(dataset) for dataset in ingest_manifests(dataset_file)["dataset"] ] raw_resources = api_helpers.list_server_resources( @@ -135,7 +135,7 @@ def annotate_dataset( existing_categories: List[str] = [resource.fides_key for resource in resources] for dataset in datasets: - current_dataset = Dataset.parse_obj(dataset) + current_dataset = Dataset.model_validate(dataset) try: click.secho(f"\n####\nAnnotating Dataset: [{current_dataset.name}]") @@ -156,14 +156,14 @@ def annotate_dataset( ) if include_null: - output_dataset.append(current_dataset.dict()) + output_dataset.append(current_dataset.model_dump(mode="json")) else: - output_dataset.append(current_dataset.dict(exclude_none=True)) + output_dataset.append(current_dataset.model_dump(exclude_none=True)) except AnnotationAbortError: if include_null: - output_dataset.append(current_dataset.dict()) + output_dataset.append(current_dataset.model_dump(mode="json")) else: - output_dataset.append(current_dataset.dict(exclude_none=True)) + output_dataset.append(current_dataset.model_dump(exclude_none=True)) break manifests.write_manifest(dataset_file, output_dataset, "dataset") echo_green("Annotation process complete.") diff --git a/src/fides/core/dataset.py b/src/fides/core/dataset.py index fc28ccd823..509828f2e0 100644 --- a/src/fides/core/dataset.py +++ b/src/fides/core/dataset.py @@ -5,9 +5,8 @@ import sqlalchemy from fideslang import manifests from fideslang.models import Dataset, DatasetCollection, DatasetField -from fideslang.validation import FidesKey +from fideslang.validation import AnyHttpUrlString, FidesKey from joblib import Parallel, delayed -from pydantic import AnyHttpUrl from sqlalchemy.engine import Engine from sqlalchemy.sql import text @@ -34,7 +33,7 @@ def get_all_server_datasets( - url: AnyHttpUrl, headers: Dict[str, str], exclude_datasets: List[Dataset] + url: AnyHttpUrlString, headers: Dict[str, str], exclude_datasets: List[Dataset] ) -> List[Dataset]: """ Get a list of all of the Datasets that exist on the server. Excludes any datasets @@ -43,14 +42,14 @@ def get_all_server_datasets( exclude_dataset_keys = [dataset.fides_key for dataset in exclude_datasets] raw_dataset_list = ( list_server_resources( - url=url, + url=str(url), resource_type="dataset", exclude_keys=[str(x) for x in exclude_dataset_keys], headers=headers, ) or [] ) - dataset_list = [Dataset.parse_obj(dataset) for dataset in raw_dataset_list] + dataset_list = [Dataset.model_validate(dataset) for dataset in raw_dataset_list] return dataset_list @@ -256,7 +255,7 @@ def scan_dataset_db( connection_string: str, manifest_dir: Optional[str], coverage_threshold: int, - url: AnyHttpUrl, + url: AnyHttpUrlString, headers: Dict[str, str], local: bool = False, ) -> None: @@ -336,7 +335,7 @@ def write_dataset_manifest( """ manifests.write_manifest( file_name, - [i.dict(exclude_none=not include_null) for i in datasets], + [i.model_dump(exclude_none=not include_null) for i in datasets], "dataset", ) echo_green(f"Generated dataset manifest written to {file_name}") diff --git a/src/fides/core/evaluate.py b/src/fides/core/evaluate.py index 18b737691e..02b4826751 100644 --- a/src/fides/core/evaluate.py +++ b/src/fides/core/evaluate.py @@ -19,8 +19,7 @@ ) from fideslang.relationships import get_referenced_missing_keys from fideslang.utils import get_resource_by_fides_key -from fideslang.validation import FidesKey -from pydantic import AnyHttpUrl +from fideslang.validation import AnyHttpUrlString, FidesKey from fides.common.utils import echo_green, echo_red, handle_cli_response, pretty_echo from fides.core import api @@ -32,7 +31,7 @@ def get_evaluation_policies( local_policies: List[Policy], evaluate_fides_key: Optional[str], - url: AnyHttpUrl, + url: AnyHttpUrlString, headers: Dict[str, str], ) -> List[Policy]: """ @@ -51,7 +50,7 @@ def get_evaluation_policies( return [local_policy_found] server_policy_found = get_server_resource( - url=url, + url=str(url), resource_type="policy", resource_key=evaluate_fides_key, headers=headers, @@ -69,7 +68,9 @@ def get_evaluation_policies( def get_all_server_policies( - url: AnyHttpUrl, headers: Dict[str, str], exclude: Optional[List[FidesKey]] = None + url: AnyHttpUrlString, + headers: Dict[str, str], + exclude: Optional[List[FidesKey]] = None, ) -> List[Policy]: """ Get a list of all of the Policies that exist on the server. @@ -79,7 +80,7 @@ def get_all_server_policies( exclude = exclude if exclude else [] ls_response = handle_cli_response( - api.ls(url=url, resource_type="policy", headers=headers), verbose=False + api.ls(url=str(url), resource_type="policy", headers=headers), verbose=False ) policy_keys = [ resource["fides_key"] @@ -87,7 +88,7 @@ def get_all_server_policies( if resource["fides_key"] not in exclude ] policy_list = get_server_resources( - url=url, resource_type="policy", headers=headers, existing_keys=policy_keys + url=str(url), resource_type="policy", headers=headers, existing_keys=policy_keys ) return policy_list # type: ignore[return-value] @@ -122,7 +123,7 @@ def get_fides_key_parent_hierarchy( ) if found_resource_map: found_resource = list(found_resource_map.values())[-1] - if found_resource and "parent_key" in found_resource.__fields_set__: + if found_resource and "parent_key" in found_resource.model_fields_set: current_key = getattr(found_resource, "parent_key") if not current_key: break @@ -430,7 +431,7 @@ def execute_evaluation(taxonomy: Taxonomy) -> Evaluation: def hydrate_missing_resources( - url: AnyHttpUrl, + url: AnyHttpUrlString, headers: Dict[str, str], missing_resource_keys: List[FidesKey], dehydrated_taxonomy: Taxonomy, @@ -440,9 +441,9 @@ def hydrate_missing_resources( hydrate a copy of the dehydrated taxonomy with them. """ - for resource_name in dehydrated_taxonomy.__fields__: + for resource_name in dehydrated_taxonomy.model_fields: server_resources = get_server_resources( - url=url, + url=str(url), resource_type=resource_name, headers=headers, existing_keys=missing_resource_keys, @@ -457,7 +458,7 @@ def hydrate_missing_resources( def populate_referenced_keys( taxonomy: Taxonomy, - url: AnyHttpUrl, + url: AnyHttpUrlString, headers: Dict[str, str], last_keys: List[FidesKey], ) -> Taxonomy: @@ -481,7 +482,10 @@ def populate_referenced_keys( dehydrated_taxonomy=taxonomy, ) return populate_referenced_keys( - taxonomy=taxonomy, url=url, headers=headers, last_keys=missing_resource_keys + taxonomy=taxonomy, + url=url, + headers=headers, + last_keys=missing_resource_keys, ) return taxonomy @@ -493,7 +497,7 @@ def merge_taxonomies( Merges the secondary_taxonomy into the primary_taxonomy while preserving all of the existing keys within the primary_taxonomy. """ - for resource_name in primary_taxonomy.__fields__: + for resource_name in primary_taxonomy.model_fields: # Get the unique set of keys we want to include in the merged taxonomy primary_keys = [ resource.fides_key for resource in getattr(primary_taxonomy, resource_name) @@ -511,7 +515,7 @@ def merge_taxonomies( def evaluate( - url: AnyHttpUrl, + url: AnyHttpUrlString, manifests_dir: str, headers: Dict[str, str], policy_fides_key: str = "", @@ -565,7 +569,7 @@ def evaluate( if not dry: echo_green("Sending the evaluation results to the server...") response = api.create( - url=url, + url=str(url), resource_type="evaluation", json_resource=evaluation.json(exclude_none=True), headers=headers, @@ -573,7 +577,7 @@ def evaluate( handle_cli_response(response, verbose=False) if evaluation.status == "FAIL": - pretty_echo(evaluation.dict(), color="red") + pretty_echo(evaluation.model_dump(mode="json"), color="red") raise SystemExit(1) echo_green("Evaluation passed!") diff --git a/src/fides/core/push.py b/src/fides/core/push.py index 6e7cb5701a..937f550104 100644 --- a/src/fides/core/push.py +++ b/src/fides/core/push.py @@ -1,6 +1,5 @@ """This module handles the logic required for pushing manifest files to the server.""" -from json import loads from pprint import pprint from typing import Dict, List, Tuple @@ -40,8 +39,8 @@ def sort_create_update( if diff: resource_diff = DeepDiff( - manifest_resource.dict(), - server_resource.dict(), + manifest_resource.model_dump(mode="json"), + server_resource.model_dump(mode="json"), ) if resource_diff: print( @@ -54,7 +53,7 @@ def sort_create_update( else: if diff: print(f"\nNew resource with fides_key: {manifest_resource.fides_key}") - pprint(manifest_resource.dict(exclude_unset=True)) + pprint(manifest_resource.model_dump(exclude_unset=True)) create_list.append(manifest_resource) return create_list, update_list @@ -114,7 +113,7 @@ def push( for dataset in missing_datasets: print(dataset) - for resource_type in taxonomy.__fields_set__: + for resource_type in taxonomy.model_fields_set: print("-" * 10) print(f"Processing {resource_type} resource(s)...") resource_list = getattr(taxonomy, resource_type) @@ -140,7 +139,9 @@ def push( headers=headers, resource_type=resource_type, url=url, - resources=[loads(resource.json()) for resource in resource_list], + resources=[ + resource.model_dump(mode="json") for resource in resource_list + ], ), verbose=False, ) diff --git a/src/fides/core/system.py b/src/fides/core/system.py index 4f2ac8c2f5..9ee64ad388 100644 --- a/src/fides/core/system.py +++ b/src/fides/core/system.py @@ -6,7 +6,7 @@ from fideslang import manifests from fideslang.models import Organization, System -from pydantic import AnyHttpUrl +from fideslang.validation import AnyHttpUrlString from fides.common.utils import echo_green, echo_red, handle_cli_response from fides.connectors.models import AWSConfig, OktaConfig @@ -72,7 +72,7 @@ def generate_resource_tagging_systems( def get_organization( organization_key: str, manifest_organizations: List[Organization], - url: AnyHttpUrl, + url: AnyHttpUrlString, headers: Dict[str, str], ) -> Optional[Organization]: """ @@ -88,7 +88,7 @@ def get_organization( return taxonomy_organization server_organization = get_server_resource( - url=url, + url=str(url), resource_type="organization", resource_key=organization_key, headers=headers, @@ -102,7 +102,7 @@ def get_organization( ) raise SystemExit(1) - parsed_organization = Organization.parse_obj(server_organization) + parsed_organization = Organization.model_validate(server_organization) assert isinstance(parsed_organization, Organization) return parsed_organization @@ -143,7 +143,7 @@ def write_system_manifest( """ manifests.write_manifest( file_name, - [i.dict(exclude_none=not include_null) for i in systems], + [i.model_dump(exclude_none=not include_null) for i in systems], "system", ) echo_green(f"Generated system manifest written to {file_name}") @@ -154,7 +154,7 @@ def generate_system_aws( include_null: bool, organization_key: str, aws_config: Optional[AWSConfig], - url: AnyHttpUrl, + url: AnyHttpUrlString, headers: Dict[str, str], ) -> str: """ @@ -202,7 +202,7 @@ def generate_system_okta( okta_config: Optional[OktaConfig], file_name: str, include_null: bool, - url: AnyHttpUrl, + url: AnyHttpUrlString, headers: Dict[str, str], ) -> str: """ @@ -228,14 +228,14 @@ def generate_system_okta( def get_all_server_systems( - url: AnyHttpUrl, headers: Dict[str, str], exclude_systems: List[System] + url: AnyHttpUrlString, headers: Dict[str, str], exclude_systems: List[System] ) -> List[System]: """ Get a list of all of the Systems that exist on the server. Excludes any systems provided in exclude_systems """ ls_response = handle_cli_response( - api.ls(url=url, resource_type="system", headers=headers), verbose=False + api.ls(url=str(url), resource_type="system", headers=headers), verbose=False ) exclude_system_keys = [system.fides_key for system in exclude_systems] system_keys = [ @@ -248,7 +248,10 @@ def get_all_server_systems( system_list = [ System.validate(x) for x in get_server_resources( - url=url, resource_type="system", headers=headers, existing_keys=system_keys + url=str(url), + resource_type="system", + headers=headers, + existing_keys=system_keys, ) ] @@ -365,7 +368,7 @@ def scan_system_aws( organization_key: str, aws_config: Optional[AWSConfig], coverage_threshold: int, - url: AnyHttpUrl, + url: AnyHttpUrlString, headers: Dict[str, str], ) -> None: """ @@ -416,7 +419,7 @@ def scan_system_okta( organization_key: str, okta_config: Optional[OktaConfig], coverage_threshold: int, - url: AnyHttpUrl, + url: AnyHttpUrlString, headers: Dict[str, str], ) -> None: """ diff --git a/src/fides/core/utils.py b/src/fides/core/utils.py index 3ea7a02634..79c76c5edf 100644 --- a/src/fides/core/utils.py +++ b/src/fides/core/utils.py @@ -83,12 +83,10 @@ def get_all_level_fields(fields: list) -> Iterator[DatasetField]: yield field if isinstance(field, dict): if field["fields"]: - for nested_field in get_all_level_fields(field["fields"]): - yield nested_field + yield from get_all_level_fields(field["fields"]) else: if field.fields: - for nested_field in get_all_level_fields(field.fields): - yield nested_field + yield from get_all_level_fields(field.fields) def get_manifest_list(manifests_dir: str) -> List[str]: @@ -170,7 +168,7 @@ def git_is_dirty(dir_to_check: str = ".") -> bool: def write_credentials_file(credentials: Credentials, credentials_path: str) -> str: """Write the user credentials file.""" with open(credentials_path, "w", encoding="utf-8") as credentials_file: - credentials_file.write(toml.dumps(credentials.dict())) + credentials_file.write(toml.dumps(credentials.model_dump(mode="json"))) return credentials_path @@ -190,7 +188,7 @@ def read_credentials_file( if not isfile(credentials_path): raise FileNotFoundError with open(credentials_path, "r", encoding="utf-8") as credentials_file: - credentials = Credentials.parse_obj(toml.load(credentials_file)) + credentials = Credentials.model_validate(toml.load(credentials_file)) return credentials diff --git a/tests/conftest.py b/tests/conftest.py index 4ef33da56d..9ceef765e0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -378,14 +378,14 @@ def resources_dict(): """ resources_dict = { "data_category": models.DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="user.custom", parent_key="user", name="Custom Data Category", description="Custom Data Category", ), "dataset": models.Dataset( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_sample_db_dataset", name="Sample DB Dataset", description="This is a Sample Database Dataset", @@ -415,13 +415,13 @@ def resources_dict(): ], ), "data_subject": models.DataSubject( - organization_fides_key=1, + organization_fides_key="1", fides_key="custom_subject", name="Custom Data Subject", description="Custom Data Subject", ), "data_use": models.DataUse( - organization_fides_key=1, + organization_fides_key="1", fides_key="custom_data_use", name="Custom Data Use", description="Custom Data Use", @@ -435,7 +435,7 @@ def resources_dict(): description="Test Organization", ), "policy": models.Policy( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_policy", name="Test Policy", version="1.3", @@ -449,7 +449,7 @@ def resources_dict(): data_subjects=models.PrivacyRule(matches="ANY", values=[]), ), "system": models.System( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_system", system_type="SYSTEM", name="Test System", @@ -1530,4 +1530,4 @@ def load_default_data_uses(db): # weirdly, only in some test scenarios, we already have the default taxonomy # loaded, in which case the create will throw an error. so we first check existence. if DataUse.get_by(db, field="name", value=data_use.name) is None: - DataUse.create(db=db, data=data_use.dict()) + DataUse.create(db=db, data=data_use.model_dump(mode="json")) diff --git a/tests/ctl/api/test_seed.py b/tests/ctl/api/test_seed.py index c14f71ee2e..439019886d 100644 --- a/tests/ctl/api/test_seed.py +++ b/tests/ctl/api/test_seed.py @@ -211,7 +211,7 @@ async def test_add_to_default_taxonomy( ) assert result.status_code == 404 - updated_default_taxonomy = DEFAULT_TAXONOMY.copy() + updated_default_taxonomy = DEFAULT_TAXONOMY.model_copy() updated_default_taxonomy.data_category.append(data_category) monkeypatch.setattr(seed, "DEFAULT_TAXONOMY", updated_default_taxonomy) @@ -232,7 +232,7 @@ async def test_does_not_override_user_changes( Loading the default taxonomy should not override user changes to their default taxonomy """ - default_category = DEFAULT_TAXONOMY.data_category[0].copy() + default_category = DEFAULT_TAXONOMY.data_category[0].model_copy() new_description = "foo description" default_category.description = new_description result = _api.update( @@ -434,7 +434,7 @@ async def test_load_default_dsr_policies( async def test_load_organizations(loguru_caplog, async_session, monkeypatch): - updated_default_taxonomy = DEFAULT_TAXONOMY.copy() + updated_default_taxonomy = DEFAULT_TAXONOMY.model_copy() current_orgs = len(updated_default_taxonomy.organization) updated_default_taxonomy.organization.append( Organization(fides_key="new_organization") @@ -452,14 +452,20 @@ class TestLoadSamples: """Tests related to load_samples""" SAMPLE_ENV_VARS = { - # Include test secrets for Postgres and Stripe, only + # Include test secrets for Postgres, Mongo, and Stripe, only "FIDES_DEPLOY__CONNECTORS__POSTGRES__HOST": "test-var-expansion", "FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT": "9090", "FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME": "test-var-db", "FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME": "test-var-user", "FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD": "test-var-password", + "FIDES_DEPLOY__CONNECTORS__POSTGRES__SSH_REQUIRED": "True", "FIDES_DEPLOY__CONNECTORS__STRIPE__DOMAIN": "test-stripe-domain", "FIDES_DEPLOY__CONNECTORS__STRIPE__API_KEY": "test-stripe-api-key", + "FIDES_DEPLOY__CONNECTORS__MONGO_HOST": "test-var-expansion", + "FIDES_DEPLOY__CONNECTORS__MONGO_PORT": "9090", + "FIDES_DEPLOY__CONNECTORS__MONGO_DEFAULTAUTHDB": "test-var-db", + "FIDES_DEPLOY__CONNECTORS__MONGO_USERNAME": "test-var-user", + "FIDES_DEPLOY__CONNECTORS__MONGO_PASSWORD": "test-var-password", } @patch.dict(os.environ, SAMPLE_ENV_VARS, clear=True) @@ -491,8 +497,8 @@ async def test_load_samples( assert len(systems) == 5 assert len(datasets) == 4 assert len(policies) == 1 - assert len(connections) == 2 - assert len(dataset_configs) == 2 + assert len(connections) == 3 + assert len(dataset_configs) == 3 assert sorted([e.fides_key for e in systems]) == [ "cookie_house", @@ -513,10 +519,12 @@ async def test_load_samples( # expected to exist; the others defined in the sample_connections.yml # will be ignored since they are missing secrets! assert sorted([e.key for e in connections]) == [ + "cookie_house_customer_database_mongodb", "cookie_house_postgresql_database", "stripe_connector", ] assert sorted([e.fides_key for e in dataset_configs]) == [ + "mongo_test", "postgres_example_test_dataset", "stripe_connector", ] @@ -586,14 +594,17 @@ async def test_load_sample_connections(self): assert False, error_message # Assert that only the connections with all their secrets are returned - assert len(connections) == 2 + assert len(connections) == 3 assert sorted([e.key for e in connections]) == [ + "cookie_house_customer_database_mongodb", "cookie_house_postgresql_database", "stripe_connector", ] # Assert that variable expansion worked as expected - postgres = [e for e in connections if e.connection_type == "postgres"][0].dict() + postgres = [e for e in connections if e.connection_type == "postgres"][ + 0 + ].model_dump(mode="json") assert postgres["secrets"]["host"] == "test-var-expansion" assert postgres["secrets"]["port"] == 9090 diff --git a/tests/ctl/cli/test_utils.py b/tests/ctl/cli/test_utils.py index bf006efc36..a3fa04b7d6 100644 --- a/tests/ctl/cli/test_utils.py +++ b/tests/ctl/cli/test_utils.py @@ -173,7 +173,7 @@ def test_returns_config_dict( org_url=input_org_url, credentials_id=input_credentials_id, ) - assert okta_config == { + assert okta_config.model_dump() == { "orgUrl": "https://dev-78908748.okta.com", "token": "redacted_override_in_tests", } @@ -192,7 +192,10 @@ def test_returns_input_dict( org_url=input_org_url, credentials_id=input_credentials_id, ) - assert okta_config == {"orgUrl": input_org_url, "token": input_token} + assert okta_config.model_dump() == { + "orgUrl": input_org_url, + "token": input_token, + } @pytest.mark.unit @@ -253,7 +256,7 @@ def test_returns_config_dict( credentials_id=input_credentials_id, ) - assert aws_config == { + assert aws_config.model_dump(mode="json") == { "aws_access_key_id": "redacted_id_override_in_tests", "aws_secret_access_key": "redacted_override_in_tests", "region_name": "us-east-1", @@ -277,7 +280,7 @@ def test_returns_input_dict( region=input_region, credentials_id=input_credentials_id, ) - assert aws_config == { + assert aws_config.model_dump(mode="json") == { "aws_access_key_id": input_access_key_id, "aws_secret_access_key": input_access_key, "region_name": input_region, @@ -302,7 +305,7 @@ def test_session_token_temporary_credentials( region=input_region, credentials_id=input_credentials_id, ) - assert aws_config == { + assert aws_config.model_dump(mode="json") == { "aws_access_key_id": input_access_key_id, "aws_secret_access_key": input_access_key, "region_name": input_region, diff --git a/tests/ctl/conftest.py b/tests/ctl/conftest.py index 8a99c2c902..db9f33e2ef 100644 --- a/tests/ctl/conftest.py +++ b/tests/ctl/conftest.py @@ -76,4 +76,4 @@ def load_default_data_uses(db): # Here we make sure our default data uses are always available for our tests, # if they're not present already. if DataUse.get_by(db, field="name", value=data_use.name) is None: - DataUse.create(db=db, data=data_use.dict()) + DataUse.create(db=db, data=data_use.model_dump(mode="json")) diff --git a/tests/ctl/core/config/test_config.py b/tests/ctl/core/config/test_config.py index 640dae2d9f..f583922de8 100644 --- a/tests/ctl/core/config/test_config.py +++ b/tests/ctl/core/config/test_config.py @@ -152,6 +152,7 @@ def test_get_config_cache() -> None: "FIDES__USER__ENCRYPTION_KEY": "test_key_one", "FIDES__CLI__SERVER_HOST": "test", "FIDES__CLI__SERVER_PORT": "8080", + "FIDES__ADMIN_UI__URL": "http://localhost:3000/", "FIDES__CREDENTIALS__POSTGRES_1__CONNECTION_STRING": "postgresql+psycopg2://fides:env_variable.com:5439/fidesctl_test", **REQUIRED_ENV_VARS, }, @@ -159,11 +160,14 @@ def test_get_config_cache() -> None: ) @pytest.mark.unit def test_config_from_env_vars() -> None: - "Test building a config from env vars." + """Test building a config from env vars.""" config = get_config() assert config.user.encryption_key == "test_key_one" - assert config.cli.server_url == "http://test:8080" + assert ( + config.cli.server_url == "http://test:8080" + ) # No trailing slash because this is constructed from components + assert config.admin_ui.url == "http://localhost:3000" # Trailing slash is removed assert ( config.credentials["postgres_1"]["connection_string"] == "postgresql+psycopg2://fides:env_variable.com:5439/fidesctl_test" @@ -243,8 +247,8 @@ def test_config_from_path() -> None: """Test reading config using the FIDES__CONFIG_PATH option.""" config = get_config() print(os.environ) - assert config.admin_ui.enabled == True - assert config.execution.require_manual_request_approval == True + assert config.admin_ui.enabled is True + assert config.execution.require_manual_request_approval is True @patch.dict( @@ -532,5 +536,5 @@ def test_get_config_ac_mode_without_tc_mode() -> None: assert ( exc.value.errors()[0]["msg"] - == "AC cannot be enabled unless TCF mode is also enabled." + == "Value error, AC cannot be enabled unless TCF mode is also enabled." ) diff --git a/tests/ctl/core/config/test_create.py b/tests/ctl/core/config/test_create.py index 6e21a249c7..996dbd9fca 100644 --- a/tests/ctl/core/config/test_create.py +++ b/tests/ctl/core/config/test_create.py @@ -96,7 +96,7 @@ def test_create_config_file_exists( fides_file_path = fides_directory / "fides.toml" with open(fides_file_path, "w", encoding="utf-8") as f: - toml.dump(config.dict(), f) + toml.dump(config.model_dump(mode="json"), f) config_path = create_config_file(config, tmp_path) diff --git a/tests/ctl/core/config/test_security_settings.py b/tests/ctl/core/config/test_security_settings.py index c3d6595098..223721cf2e 100644 --- a/tests/ctl/core/config/test_security_settings.py +++ b/tests/ctl/core/config/test_security_settings.py @@ -4,7 +4,7 @@ @pytest.mark.unit -class TestSecuirtySettings: +class TestSecuritySettings: def test_validate_encryption_key_length_default_value(self): settings = SecuritySettings() assert settings.app_encryption_key == "" @@ -36,7 +36,7 @@ def test_validate_assemble_cors_origins_string_of_urls(self): def test_validate_cors_origins_urls_with_paths(self): with pytest.raises(ValueError) as e: - SecuritySettings(cors_origins=["http://test.com/"]) + SecuritySettings(cors_origins=["http://test.com/longerpath"]) assert "URL origin values cannot contain a path." in str(e) @@ -45,6 +45,10 @@ def test_validate_cors_origins_urls_with_paths(self): assert "URL origin values cannot contain a path." in str(e) + # If there is a trailing slash, it is now stripped off + settings = SecuritySettings(cors_origins=["http://test.com/"]) + assert settings.cors_origins == ["http://test.com"] + def test_assemble_root_access_token_none(self): settings = SecuritySettings(oauth_root_client_secret="") diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index 83175314ff..36801ddac2 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -2,7 +2,7 @@ """Integration tests for the API module.""" import json import typing -from datetime import datetime +from datetime import datetime, timezone from json import loads from typing import Dict, List, Tuple from uuid import uuid4 @@ -224,7 +224,7 @@ async def test_create_dataset_data_categories_validated( assert result.status_code == 422 assert ( result.json()["detail"][0]["msg"] - == "The data category bad_category is not supported." + == "Value error, The data category bad_category is not supported." ) @pytest.mark.parametrize("endpoint", model_list) @@ -368,7 +368,7 @@ async def test_update_dataset_data_categories_validated( assert result.status_code == 422 assert ( result.json()["detail"][0]["msg"] - == "The data category bad_category is not supported." + == "Value error, The data category bad_category is not supported." ) @pytest.mark.parametrize("endpoint", model_list) @@ -421,7 +421,7 @@ async def test_upsert_validates_resources_against_pydantic_model( ): endpoint = "dataset" manifest: Dataset = resources_dict[endpoint] - dict_manifest = manifest.dict() + dict_manifest = manifest.model_dump(mode="json") del dict_manifest["organization_fides_key"] result = _api.upsert( @@ -440,7 +440,7 @@ async def test_upsert_dataset_data_categories_validated( ): endpoint = "dataset" manifest: Dataset = resources_dict[endpoint] - dict_manifest = manifest.dict() + dict_manifest = manifest.model_dump(mode="json") dict_manifest["collections"][0]["data_categories"] = ["bad_category"] result = _api.upsert( @@ -452,7 +452,7 @@ async def test_upsert_dataset_data_categories_validated( assert result.status_code == 422 assert ( result.json()["detail"][0]["msg"] - == "The data category bad_category is not supported." + == "Value error, The data category bad_category is not supported." ) @pytest.mark.parametrize("endpoint", model_list) @@ -517,7 +517,7 @@ def remove_all_systems(self, db) -> None: @pytest.fixture(scope="function") def system_create_request_body(self) -> SystemSchema: return SystemSchema( - organization_fides_key=1, + organization_fides_key="1", fides_key="system_fides_key", system_type="SYSTEM", name="Test System", @@ -887,7 +887,7 @@ async def test_system_create( assert len(systems) == 1 system = systems[0] - for field in SystemResponse.__fields__: + for field in SystemResponse.model_fields: system_val = getattr(system, field) if isinstance(system_val, typing.Hashable) and not isinstance( system_val, datetime @@ -897,7 +897,7 @@ async def test_system_create( assert json_results["created_at"] for i, decl in enumerate(system.privacy_declarations): - for field in PrivacyDeclarationResponse.__fields__: + for field in PrivacyDeclarationResponse.model_fields: decl_val = getattr(decl, field) if isinstance(decl_val, typing.Hashable): assert decl_val == json_results["privacy_declarations"][i][field] @@ -1585,10 +1585,11 @@ def remove_all_systems(self, db) -> None: @pytest.fixture(scope="function") def system_update_request_body(self, system) -> SystemSchema: return SystemSchema( - organization_fides_key=1, + organization_fides_key="1", fides_key=system.fides_key, system_type="SYSTEM", name=self.updated_system_name, + vendor_deleted_date=datetime(2022, 5, 22), description="Test Policy", privacy_declarations=[ models.PrivacyDeclaration( @@ -1606,7 +1607,7 @@ def system_update_request_body(self, system) -> SystemSchema: @pytest.fixture(scope="function") def system_update_request_body_with_system_cookies(self, system) -> SystemSchema: return SystemSchema( - organization_fides_key=1, + organization_fides_key="1", fides_key=system.fides_key, system_type="SYSTEM", name=self.updated_system_name, @@ -1633,7 +1634,7 @@ def system_update_request_body_with_privacy_declaration_cookies( self, system ) -> SystemSchema: return SystemSchema( - organization_fides_key=1, + organization_fides_key="1", fides_key=system.fides_key, system_type="SYSTEM", name=self.updated_system_name, @@ -1658,7 +1659,7 @@ def system_update_request_body_with_new_dictionary_fields( self, system ) -> SystemSchema: return SystemSchema( - organization_fides_key=1, + organization_fides_key="1", fides_key=system.fides_key, system_type="SYSTEM", name=self.updated_system_name, @@ -1839,6 +1840,7 @@ def test_system_update_as_system_manager( db.refresh(system) assert system.name == self.updated_system_name + assert system.vendor_deleted_date == datetime(2022, 5, 22, tzinfo=timezone.utc) def test_system_update_as_system_manager_403_if_not_found( self, @@ -2279,7 +2281,7 @@ def test_system_update_dictionary_fields( assert privacy_decl.shared_categories == ["user"] json_results = result.json() - for field in SystemResponse.__fields__: + for field in SystemResponse.model_fields: system_val = getattr(system, field) if isinstance(system_val, typing.Hashable) and not isinstance( system_val, datetime @@ -2290,7 +2292,7 @@ def test_system_update_dictionary_fields( assert json_results["created_at"] for i, decl in enumerate(system.privacy_declarations): - for field in PrivacyDeclarationResponse.__fields__: + for field in PrivacyDeclarationResponse.model_fields: decl_val = getattr(decl, field, None) if hasattr(decl, field) and isinstance(decl_val, typing.Hashable): assert decl_val == json_results["privacy_declarations"][i][field] @@ -2518,7 +2520,7 @@ def test_system_update_updates_declarations( "id" in response_dec.keys() ), "No 'id' field in the response declaration!" - parsed_response_declaration = models.PrivacyDeclaration.parse_obj( + parsed_response_declaration = models.PrivacyDeclaration.model_validate( response_dec ) assert ( @@ -2531,12 +2533,12 @@ def test_system_update_updates_declarations( system = System.all(db)[0] db.refresh(system) db_decs = [ - models.PrivacyDeclaration.from_orm(db_dec) + models.PrivacyDeclaration.model_validate(db_dec) for db_dec in system.privacy_declarations ] for update_dec in update_declarations: - db_decs.remove(update_dec.dict()) + db_decs.remove(update_dec) # and assert we don't have any extra response declarations assert len(db_decs) == 0 @@ -2709,7 +2711,7 @@ def test_system_update_manages_declaration_records( Test to assert that existing privacy declaration records stay constant when necessary """ old_db_decs = [ - PrivacyDeclarationResponse.from_orm(dec) + PrivacyDeclarationResponse.model_validate(dec) for dec in system_multiple_decs.privacy_declarations ] old_decs_updated = [ @@ -2965,7 +2967,9 @@ def test_api_can_upsert_default( self, test_config: FidesConfig, endpoint: str ) -> None: """Should be able to upsert as long as `is_default` is not changing""" - resources = [r.dict() for r in getattr(DEFAULT_TAXONOMY, endpoint)[0:2]] + resources = [ + r.model_dump(mode="json") for r in getattr(DEFAULT_TAXONOMY, endpoint)[0:2] + ] result = _api.upsert( url=test_config.cli.server_url, headers=test_config.user.auth_header, @@ -3017,7 +3021,7 @@ def test_api_cannot_upsert_default_taxonomy( url=test_config.cli.server_url, headers=test_config.user.auth_header, resource_type=endpoint, - resources=[manifest.dict()], + resources=[manifest.model_dump(mode="json")], ) assert result.status_code == 403 assert ( @@ -3065,7 +3069,7 @@ def test_api_cannot_upsert_is_default( self, test_config: FidesConfig, resources_dict: Dict, endpoint: str ) -> None: manifest = resources_dict[endpoint] - second_item = manifest.copy() + second_item = manifest.model_copy() # Set fields for default labels manifest.is_default = True @@ -3084,7 +3088,10 @@ def test_api_cannot_upsert_is_default( url=test_config.cli.server_url, headers=test_config.user.auth_header, resource_type=endpoint, - resources=[manifest.dict(), second_item.dict()], + resources=[ + manifest.model_dump(mode="json"), + second_item.model_dump(mode="json"), + ], ) assert result.status_code == 403 assert ( @@ -3121,7 +3128,7 @@ def test_api_can_update_active_on_default( """Ensure we can toggle `active` property on default taxonomy elements""" resource = getattr(DEFAULT_TAXONOMY, endpoint)[0] resource = TAXONOMY_EXTENSIONS[endpoint]( - **resource.dict() + **resource.model_dump(mode="json") ) # cast resource to extended model resource.active = False json_resource = resource.json(exclude_none=True) @@ -3172,7 +3179,7 @@ def test_api_can_create_with_active_property( # get a default taxonomy element as a sample resource resource = getattr(DEFAULT_TAXONOMY, endpoint)[0] resource = TAXONOMY_EXTENSIONS[endpoint]( - **resource.dict() + **resource.model_dump(mode="json") ) # cast resource to extended model resource.fides_key = resource.fides_key + "_test_create_active_false" resource.is_default = False diff --git a/tests/ctl/core/test_api_helpers.py b/tests/ctl/core/test_api_helpers.py index 38d9022a2e..62ee7ed9e7 100644 --- a/tests/ctl/core/test_api_helpers.py +++ b/tests/ctl/core/test_api_helpers.py @@ -31,7 +31,7 @@ def created_resources( created_keys = [] resource_type = request.param for _ in range(RESOURCE_CREATION_COUNT): - base_resource = resources_dict[resource_type].copy() + base_resource = resources_dict[resource_type].model_copy() base_resource.fides_key = "{}_{}".format( base_resource.fides_key, str(uuid.uuid4())[:6] ) diff --git a/tests/ctl/core/test_apply.py b/tests/ctl/core/test_apply.py index 46d7eb3da2..39f47f196e 100644 --- a/tests/ctl/core/test_apply.py +++ b/tests/ctl/core/test_apply.py @@ -26,7 +26,7 @@ def server_resource_key_pairs() -> Generator: def system_with_dataset_reference() -> Generator: yield [ models.System( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_system", system_type="test", privacy_declarations=[ @@ -46,13 +46,13 @@ def system_with_dataset_reference() -> Generator: @pytest.mark.unit def test_sort_create_update_create() -> None: resource_1 = models.DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="some_resource", name="Test resource 1", description="Test Description", ) resource_2 = models.DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="another_system", name="Test System 2", description="Test Description", @@ -73,13 +73,13 @@ def test_sort_create_update_create() -> None: def test_sort_create_update_update() -> None: resource_1 = models.DataCategory( id=1, - organization_fides_key=1, + organization_fides_key="1", fides_key="some_resource", name="Test resource 1", description="Test Description", ) resource_2 = models.DataCategory( - organization_fides_key=1, + organization_fides_key="1", fides_key="some_resource", name="Test System 2", description="Test Description", @@ -103,7 +103,7 @@ def test_sort_create_update_update() -> None: { "dataset": [ models.Dataset( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_dataset", collections=[], ), @@ -115,7 +115,7 @@ def test_sort_create_update_update() -> None: { "dataset": [ models.Dataset( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_dataset_unused", collections=[], ), @@ -127,12 +127,12 @@ def test_sort_create_update_update() -> None: { "dataset": [ models.Dataset( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_dataset", collections=[], ), models.Dataset( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_dataset_unused", collections=[], ), @@ -144,17 +144,17 @@ def test_sort_create_update_update() -> None: { "dataset": [ models.Dataset( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_dataset", collections=[], ), models.Dataset( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_dataset_unused", collections=[], ), models.Dataset( - organization_fides_key=1, + organization_fides_key="1", fides_key="test_dataset_also_unused", collections=[], ), @@ -172,6 +172,6 @@ def test_validate_dataset_usage( Tests different scenarios for referenced datasets """ taxonomies["system"] = system_with_dataset_reference - taxonomy = models.Taxonomy.parse_obj(taxonomies) + taxonomy = models.Taxonomy.model_validate(taxonomies) missing_datasets = get_orphan_datasets(taxonomy) assert len(missing_datasets) == expected_length diff --git a/tests/ctl/core/test_dataset.py b/tests/ctl/core/test_dataset.py index 1635653323..1786978333 100644 --- a/tests/ctl/core/test_dataset.py +++ b/tests/ctl/core/test_dataset.py @@ -65,12 +65,12 @@ def test_create_db_datasets() -> None: data_categories=[], fields=[ DatasetField( - name=1, + name="1", description="Fides Generated Description for Column: 1", data_categories=[], ), DatasetField( - name=2, + name="2", description="Fides Generated Description for Column: 2", data_categories=[], ), @@ -82,12 +82,12 @@ def test_create_db_datasets() -> None: data_categories=[], fields=[ DatasetField( - name=4, + name="4", description="Fides Generated Description for Column: 4", data_categories=[], ), DatasetField( - name=5, + name="5", description="Fides Generated Description for Column: 5", data_categories=[], ), @@ -112,11 +112,11 @@ def test_find_uncategorized_dataset_fields_all_categorized() -> None: name="foo", fields=[ DatasetField( - name=1, + name="1", data_categories=["category_1"], ), DatasetField( - name=2, + name="2", data_categories=["category_1"], ), ], @@ -125,10 +125,10 @@ def test_find_uncategorized_dataset_fields_all_categorized() -> None: name="bar", fields=[ DatasetField( - name=4, + name="4", data_categories=["category_1"], ), - DatasetField(name=5, data_categories=["category_1"]), + DatasetField(name="5", data_categories=["category_1"]), ], ), ], @@ -184,12 +184,12 @@ async def test_upsert_db_datasets( data_categories=[], fields=[ DatasetField( - name=1, + name="1", description="Fides Generated Description for Column: 1", data_categories=[], ), DatasetField( - name=2, + name="2", description="Fides Generated Description for Column: 2", data_categories=[], ), @@ -201,12 +201,12 @@ async def test_upsert_db_datasets( data_categories=[], fields=[ DatasetField( - name=4, + name="4", description="Fides Generated Description for Column: 4", data_categories=[], ), DatasetField( - name=5, + name="5", description="Fides Generated Description for Column: 5", data_categories=[], ), @@ -218,7 +218,7 @@ async def test_upsert_db_datasets( resp = api.upsert( url=test_config.cli.server_url, resource_type="dataset", - resources=[dataset.dict(exclude_none=True)], + resources=[dataset.model_dump(exclude_none=True)], headers=test_config.user.auth_header, ) assert resp.status_code == 201 @@ -244,7 +244,7 @@ async def test_upsert_db_datasets( resp = api.upsert( url=test_config.cli.server_url, resource_type="dataset", - resources=[dataset.dict(exclude_none=True)], + resources=[dataset.model_dump(exclude_none=True)], headers=test_config.user.auth_header, ) assert resp.status_code == 200 @@ -270,10 +270,10 @@ def test_find_uncategorized_dataset_fields_uncategorized_fields() -> None: data_categories=["category_1"], fields=[ DatasetField( - name=1, + name="1", data_categories=["category_1"], ), - DatasetField(name=2), + DatasetField(name="2"), ], ) ], @@ -300,7 +300,7 @@ def test_find_uncategorized_dataset_fields_missing_field() -> None: name="bar", fields=[ DatasetField( - name=4, + name="4", data_categories=["category_1"], ) ], @@ -329,11 +329,11 @@ def test_find_uncategorized_dataset_fields_missing_collection() -> None: name="bar", fields=[ DatasetField( - name=4, + name="4", data_categories=["category_1"], ), DatasetField( - name=5, + name="5", data_categories=["category_1"], ), ], @@ -570,7 +570,9 @@ def test_dataset_coverage_manifest_passes( set_field_data_categories(datasets, "system.operations") file_name = tmpdir.join("dataset.yml") - write_manifest(file_name, [i.dict() for i in datasets], "dataset") + write_manifest( + file_name, [i.model_dump(mode="json") for i in datasets], "dataset" + ) create_server_datasets(test_config, datasets) _dataset.scan_dataset_db( diff --git a/tests/ctl/core/test_evaluate.py b/tests/ctl/core/test_evaluate.py index dd0d468637..eea0c0c8d5 100644 --- a/tests/ctl/core/test_evaluate.py +++ b/tests/ctl/core/test_evaluate.py @@ -176,7 +176,7 @@ def test_hydrate_missing_resources(test_config: FidesConfig) -> None: ), ], system=[ - System.construct( + System.model_construct( name="test_dc", fides_key="test_dc", description="test description", @@ -555,7 +555,7 @@ def test_failed_evaluation_error_message( expected_error_message = string_cleaner( """ 'message': '', - 'status': , + 'status': 'FAIL', 'violations': [ { 'detail': 'Declaration (Share Political Opinions) of ' 'system (customer_data_sharing_system) failed ' 'rule (reject_targeted_marketing) from policy ' diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index de39d67290..8761c96915 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -133,7 +133,7 @@ def redshift_describe_clusters() -> Generator: @pytest.fixture() def redshift_systems() -> Generator: redshift_systems = [ - System.construct( + System.model_construct( fides_key="redshift-cluster-1", organization_fides_key="default_organization", name="redshift-cluster-1", @@ -146,7 +146,7 @@ def redshift_systems() -> Generator: system_type="redshift_cluster", privacy_declarations=[], ), - System.construct( + System.model_construct( fides_key="redshift-cluster-2", organization_fides_key="default_organization", name="redshift-cluster-2", @@ -186,7 +186,7 @@ def rds_systems() -> Generator: description="Fides Generated Description for RDS Instance: database-1", fidesctl_meta=SystemMetadata( endpoint_address="database-1.ckrdpkkb4ukm.us-east-1.rds.amazonaws.com", - endpoint_port="3306", + endpoint_port=3306, # This is converted to a string via model_config = ConfigDict(coerce_numbers_to_str=True) resource_id="arn:aws:rds:us-east-1:910934740016:db:database-1", ), system_type="rds_instance", diff --git a/tests/ctl/core/test_system_history.py b/tests/ctl/core/test_system_history.py index d3e3680315..3ab365fc8b 100644 --- a/tests/ctl/core/test_system_history.py +++ b/tests/ctl/core/test_system_history.py @@ -36,7 +36,7 @@ async def system(self, async_session_temp): async def test_system_information_changes( self, db, async_session_temp, system: System ): - system_schema = SystemSchema.from_orm(system) + system_schema = SystemSchema.model_validate(system) system_schema.description = "Test system" _, updated = await update_system( system_schema, async_session_temp, CONFIG.security.oauth_root_client_id @@ -52,7 +52,7 @@ async def test_system_information_changes( async def test_privacy_declaration_changes( self, db, async_session_temp, system: System ): - system_schema = SystemSchema.from_orm(system) + system_schema = SystemSchema.model_validate(system) system_schema.privacy_declarations = ( PrivacyDeclarationSchema( name="declaration-name", @@ -74,7 +74,7 @@ async def test_privacy_declaration_changes( assert system_histories[0].edited_by == CONFIG.security.root_username async def test_ingress_egress_changes(self, db, async_session_temp, system: System): - system_schema = SystemSchema.from_orm(system) + system_schema = SystemSchema.model_validate(system) system_schema.ingress = [DataFlowSchema(fides_key="upstream", type="system")] system_schema.egress = [DataFlowSchema(fides_key="user", type="user")] _, updated = await update_system( @@ -88,7 +88,7 @@ async def test_ingress_egress_changes(self, db, async_session_temp, system: Syst assert len(system_histories) == 1 async def test_multiple_changes(self, db, async_session_temp, system: System): - system_schema = SystemSchema.from_orm(system) + system_schema = SystemSchema.model_validate(system) system_schema.description = "Test system" system_schema.privacy_declarations = ( PrivacyDeclarationSchema( @@ -114,7 +114,7 @@ async def test_multiple_changes(self, db, async_session_temp, system: System): assert system_history.edited_by == CONFIG.security.root_username async def test_no_changes(self, db, async_session_temp, system: System): - system_schema = SystemSchema.from_orm(system) + system_schema = SystemSchema.model_validate(system) _, updated = await update_system( system_schema, async_session_temp, CONFIG.security.oauth_root_client_id ) @@ -131,7 +131,7 @@ async def test_automatic_system_update( self, db, async_session_temp, system: System ): """If user id doesn't map to a user in the db or the root user, we just return the original user string""" - system_schema = SystemSchema.from_orm(system) + system_schema = SystemSchema.model_validate(system) system_schema.description = "Test system" updated_system, updated = await update_system( system_schema, async_session_temp, "automatic_system_update" diff --git a/tests/ctl/core/test_user.py b/tests/ctl/core/test_user.py index 8dfe8d7e62..90831cfd8c 100644 --- a/tests/ctl/core/test_user.py +++ b/tests/ctl/core/test_user.py @@ -1,5 +1,4 @@ from os import environ -from unittest.mock import patch import pytest diff --git a/tests/ctl/core/test_utils.py b/tests/ctl/core/test_utils.py index eca98f3e44..8f803e9456 100644 --- a/tests/ctl/core/test_utils.py +++ b/tests/ctl/core/test_utils.py @@ -57,7 +57,9 @@ def test_nested_fields_unpacked( """ collection = test_nested_collection_fields collected_field_names = [] - for field in core_utils.get_all_level_fields(collection.dict()["fields"]): + for field in core_utils.get_all_level_fields( + collection.model_dump(mode="json")["fields"] + ): collected_field_names.append(field["name"]) assert len(collected_field_names) == 5 diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index e8f7f950b3..f520b7bbe7 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -371,7 +371,7 @@ def property_a(db) -> Generator: experiences=[], messaging_templates=[], paths=["test"], - ).dict(), + ).model_dump(), ) yield prop_a prop_a.delete(db=db) @@ -387,7 +387,7 @@ def property_b(db: Session) -> Generator: experiences=[], messaging_templates=[], paths=[], - ).dict(), + ).model_dump(), ) yield prop_b prop_b.delete(db=db) @@ -2351,7 +2351,7 @@ def ctl_dataset(db: Session, example_datasets): }, ], ) - dataset = CtlDataset(**ds.dict()) + dataset = CtlDataset(**ds.model_dump(mode="json")) db.add(dataset) db.commit() yield dataset @@ -2384,7 +2384,7 @@ def unlinked_dataset(db: Session): }, ], ) - dataset = CtlDataset(**ds.dict()) + dataset = CtlDataset(**ds.model_dump(mode="json")) db.add(dataset) db.commit() yield dataset @@ -2417,7 +2417,7 @@ def linked_dataset(db: Session, connection_config: ConnectionConfig) -> Generato }, ], ) - dataset = CtlDataset(**ds.dict()) + dataset = CtlDataset(**ds.model_dump(mode="json")) db.add(dataset) db.commit() dataset_config = DatasetConfig.create( diff --git a/tests/fixtures/bigquery_fixtures.py b/tests/fixtures/bigquery_fixtures.py index 5cafbc44ec..4f9014d570 100644 --- a/tests/fixtures/bigquery_fixtures.py +++ b/tests/fixtures/bigquery_fixtures.py @@ -54,7 +54,7 @@ def bigquery_connection_config(db: Session) -> Generator: ) if keyfile_creds: schema = BigQuerySchema(keyfile_creds=keyfile_creds, dataset=dataset) - connection_config.secrets = schema.dict() + connection_config.secrets = schema.model_dump(mode="json") connection_config.save(db=db) yield connection_config @@ -159,7 +159,7 @@ def bigquery_test_engine() -> Generator: ) if keyfile_creds: schema = BigQuerySchema(keyfile_creds=keyfile_creds, dataset=dataset) - connection_config.secrets = schema.dict() + connection_config.secrets = schema.model_dump(mode="json") connector: BigQueryConnector = get_connector(connection_config) engine = connector.client() diff --git a/tests/fixtures/dynamodb_fixtures.py b/tests/fixtures/dynamodb_fixtures.py index 4da4e6b695..bdb0ca2bdd 100644 --- a/tests/fixtures/dynamodb_fixtures.py +++ b/tests/fixtures/dynamodb_fixtures.py @@ -65,7 +65,7 @@ def dynamodb_connection_config( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, ) - dynamodb_connection_config.secrets = schema.dict() + dynamodb_connection_config.secrets = schema.model_dump(mode="json") dynamodb_connection_config.save(db=db) yield dynamodb_connection_config diff --git a/tests/fixtures/google_cloud_sql_mysql_fixtures.py b/tests/fixtures/google_cloud_sql_mysql_fixtures.py index 2ca8602e01..907c1f3618 100644 --- a/tests/fixtures/google_cloud_sql_mysql_fixtures.py +++ b/tests/fixtures/google_cloud_sql_mysql_fixtures.py @@ -59,7 +59,7 @@ def google_cloud_sql_mysql_connection_config(db: Session) -> Generator: dbname=dbname, keyfile_creds=keyfile_creds, ) - connection_config.secrets = schema.dict() + connection_config.secrets = schema.model_dump(mode="json") connection_config.save(db=db) yield connection_config diff --git a/tests/fixtures/google_cloud_sql_postgres_fixtures.py b/tests/fixtures/google_cloud_sql_postgres_fixtures.py index 359c9ad3fd..c6f370550b 100644 --- a/tests/fixtures/google_cloud_sql_postgres_fixtures.py +++ b/tests/fixtures/google_cloud_sql_postgres_fixtures.py @@ -64,7 +64,7 @@ def google_cloud_sql_postgres_connection_config(db: Session) -> Generator: db_schema=db_schema, keyfile_creds=keyfile_creds, ) - connection_config.secrets = schema.dict() + connection_config.secrets = schema.model_dump() connection_config.save(db=db) yield connection_config diff --git a/tests/fixtures/redshift_fixtures.py b/tests/fixtures/redshift_fixtures.py index bf24764af5..6e9e16df1f 100644 --- a/tests/fixtures/redshift_fixtures.py +++ b/tests/fixtures/redshift_fixtures.py @@ -59,7 +59,7 @@ def redshift_connection_config(db: Session) -> Generator: database=database, db_schema=db_schema, ) - connection_config.secrets = schema.dict() + connection_config.secrets = schema.model_dump(mode="json") connection_config.save(db=db) yield connection_config diff --git a/tests/fixtures/saas/connection_template_fixtures.py b/tests/fixtures/saas/connection_template_fixtures.py index 34a65f3f2b..442dc69b7f 100644 --- a/tests/fixtures/saas/connection_template_fixtures.py +++ b/tests/fixtures/saas/connection_template_fixtures.py @@ -127,7 +127,7 @@ def instantiate_connector( connection_config.secrets = validate_secrets( db, template_vals.secrets, connection_config - ).dict() + ).model_dump(mode="json") connection_config.save(db=db) # Not persisted to db until secrets are validated dataset_config: DatasetConfig = upsert_dataset_config_from_template( diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index 7b3859ac01..06c60c7777 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -211,7 +211,7 @@ def saas_example_connection_config_with_invalid_saas_config( # replace with placholder identity invalid_saas_config["endpoints"][6]["requests"]["read"]["param_values"].pop() invalid_saas_config["endpoints"][6]["requests"]["read"]["param_values"].append( - ParamValue(name="placeholder", identity="email").dict() + ParamValue(name="placeholder", identity="email").model_dump(mode="json") ) invalid_saas_config["endpoints"][6]["requests"]["update"]["param_values"].pop() diff --git a/tests/fixtures/snowflake_fixtures.py b/tests/fixtures/snowflake_fixtures.py index 49829671d6..5b5c685f1e 100644 --- a/tests/fixtures/snowflake_fixtures.py +++ b/tests/fixtures/snowflake_fixtures.py @@ -87,7 +87,7 @@ def snowflake_connection_config( database_name=database_name, schema_name=schema_name, ) - connection_config.secrets = schema.dict() + connection_config.secrets = schema.model_dump(mode="json") connection_config.save(db=db) yield connection_config @@ -148,7 +148,7 @@ def snowflake_connection_config_with_keypair( database_name=database_name, schema_name=schema_name, ) - connection_config.secrets = schema.dict() + connection_config.secrets = schema.model_dump() connection_config.save(db=db) yield connection_config diff --git a/tests/lib/test_custom_types.py b/tests/lib/test_custom_types.py index 2d6669a309..8f963bc140 100644 --- a/tests/lib/test_custom_types.py +++ b/tests/lib/test_custom_types.py @@ -1,6 +1,13 @@ import pytest -from fides.api.custom_types import GPPMechanismConsentValue, PhoneNumber, SafeStr +from fides.api.custom_types import ( + GPPMechanismConsentValue, + PhoneNumber, + SafeStr, + validate_gpp_mechanism_consent_value, + validate_phone_number, + validate_safe_str, +) DANGEROUS_STRINGS = [ "", @@ -127,7 +134,7 @@ def test_danger_list(self, dangerous_string: str) -> None: Validate that whatever dangerous strings are being input, a sanitized/changed string is being passed back. """ - result = SafeStr.validate(dangerous_string) + result = validate_safe_str(dangerous_string) assert result != dangerous_string @@ -137,12 +144,12 @@ class TestPhoneNumber: def test_invalid_phone_numbers(self, phone_number: str) -> None: """Test that a list of invalid phone numbers is caught.""" with pytest.raises(ValueError): - PhoneNumber.validate(phone_number) + validate_phone_number(phone_number) @pytest.mark.parametrize("phone_number", VALID_PHONE_NUMBER_LIST) def test_valid_phone_numbers(self, phone_number: str) -> None: """Test that a list of valid phone numbers doesn't throw any errors.""" - validated_number = PhoneNumber.validate(phone_number) + validated_number = validate_phone_number(phone_number) assert validated_number == phone_number @@ -177,10 +184,10 @@ class TestGppMechanismConsentValue: def test_invalid_gpp_mechanism_consent_values(self, consent_value: str) -> None: """Test that invalid GPP consent mechanism values are caught.""" with pytest.raises(ValueError): - GPPMechanismConsentValue.validate(consent_value) + validate_gpp_mechanism_consent_value(consent_value) @pytest.mark.parametrize("consent_value", VALID_GPP_MECHANISM_CONSENT_VALUES) def test_valid_phone_numbers(self, consent_value: str) -> None: """Test that valid GPP consent mechanism values do not throw any errors.""" - validated_value = GPPMechanismConsentValue.validate(consent_value) + validated_value = validate_gpp_mechanism_consent_value(consent_value) assert validated_value == validated_value diff --git a/tests/lib/test_schema_base_class.py b/tests/lib/test_schema_base_class.py index 3644105706..8a5a3e6e88 100644 --- a/tests/lib/test_schema_base_class.py +++ b/tests/lib/test_schema_base_class.py @@ -1,6 +1,7 @@ # pylint: disable=missing-function-docstring from fides.api.schemas.base_class import FidesSchema, NoValidationSchema +from fides.api.schemas.connection_configuration import MongoDBDocsSchema def test_get_field_names(): @@ -14,4 +15,17 @@ class Test(FidesSchema): def test_no_validation(): test_value = "test" - assert NoValidationSchema.validate(test_value) == test_value + assert NoValidationSchema.model_validate(test_value) == test_value + + mongo_secrets = { + "host": "mongodb-test", + "port": 27017, + "username": "mongo_user", + "password": "mongo_pass", + "defaultauthdb": "mongo_test", + } + + mongo_schema = MongoDBDocsSchema.model_validate(mongo_secrets) + + assert mongo_schema == mongo_secrets + assert isinstance(mongo_secrets, dict) diff --git a/tests/ops/api/v1/endpoints/test_config_endpoints.py b/tests/ops/api/v1/endpoints/test_config_endpoints.py index 87cc511503..13000ca8a3 100644 --- a/tests/ops/api/v1/endpoints/test_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_config_endpoints.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy from typing import Any, Generator import pytest @@ -31,13 +32,13 @@ def original_cors_middleware_origins() -> Generator: assert original_cors_middleware is not None - yield original_cors_middleware.options["allow_origins"] + yield original_cors_middleware.kwargs["allow_origins"] # on teardown - find the new cors middleware, remove it, and add back in the original update_cors_middleware( app, - original_cors_middleware.options["allow_origins"], - original_cors_middleware.options["allow_origin_regex"], + original_cors_middleware.kwargs["allow_origins"], + original_cors_middleware.kwargs.get("allow_origin_regex"), ) @@ -62,11 +63,12 @@ def payload(self): }, "security": { "cors_origins": [ - "http://acme1.example.com", + "http://acme1.example.com/", "http://acme2.example.com", "http://acme3.example.com", ] }, + "admin_ui": {"enabled": True, "url": "http://localhost:3000/"}, } def test_patch_application_config_unauthenticated( @@ -201,11 +203,35 @@ def test_patch_application_config( assert response_settings["storage"] == payload["storage"] assert response_settings["execution"] == payload["execution"] assert response_settings["notifications"] == payload["notifications"] + + admin_ui_payload = copy.deepcopy(payload["admin_ui"]) + admin_ui_payload["url"] = admin_ui_payload["url"].rstrip("/") + assert ( + response_settings["admin_ui"] == admin_ui_payload + ) # Verify trailing slash stripped db_settings = db.query(ApplicationConfig).first() assert db_settings.api_set["storage"] == payload["storage"] assert db_settings.api_set["execution"] == payload["execution"] assert db_settings.api_set["notifications"] == payload["notifications"] - assert db_settings.api_set["security"] == payload["security"] + assert db_settings.api_set["admin_ui"] == admin_ui_payload + + # Security payload - cors origins had their trailing slashes removed + security_payload = copy.deepcopy(payload["security"]) + security_payload["cors_origins"] = [ + url.rstrip("/") for url in security_payload["cors_origins"] + ] + assert db_settings.api_set["security"] == security_payload + + response = api_client.get( + urls.V1_URL_PREFIX + urls.CONFIG, + headers=generate_auth_header([scopes.CONFIG_READ]), + params={"api_set": True}, + ) + # No trailing slash on url + assert response.json()["admin_ui"] == { + "enabled": True, + "url": "http://localhost:3000", + } # try PATCHing a single property updated_payload = {"storage": {"active_default_storage_type": "local"}} @@ -226,7 +252,7 @@ def test_patch_application_config( # but other properties were not impacted assert db_settings.api_set["execution"] == payload["execution"] assert db_settings.api_set["notifications"] == payload["notifications"] - assert db_settings.api_set["security"] == payload["security"] + assert db_settings.api_set["security"] == security_payload # try PATCHing multiple properties in the same nested object updated_payload = { @@ -260,7 +286,7 @@ def test_patch_application_config( response_settings["notifications"]["send_request_receipt_notification"] is True ) - assert db_settings.api_set["security"] == payload["security"] + assert db_settings.api_set["security"] == security_payload db.refresh(db_settings) # ensure property was updated on backend @@ -282,7 +308,7 @@ def test_patch_application_config( db_settings.api_set["notifications"]["send_request_receipt_notification"] is True ) - assert db_settings.api_set["security"] == payload["security"] + assert db_settings.api_set["security"] == security_payload def test_patch_application_config_notifications_properties( self, @@ -328,9 +354,12 @@ def test_patch_application_config_updates_cors_domains_in_middleware( current_cors_middleware = find_cors_middleware(api_client.app) - assert set(current_cors_middleware.options["allow_origins"]) == set( - payload["security"]["cors_origins"] - ).union(set(original_cors_middleware_origins)) + requested_origins = { + url.rstrip("/") for url in set(payload["security"]["cors_origins"]) + } + assert set( + current_cors_middleware.kwargs["allow_origins"] + ) == requested_origins.union(set(original_cors_middleware_origins)) # now ensure that the middleware update was effective by trying # some sample requests with different origins @@ -342,13 +371,14 @@ def test_patch_application_config_updates_cors_domains_in_middleware( "Origin": original_cors_middleware_origins[0], "Access-Control-Request-Method": "PATCH", } + response = api_client.options(url, headers=headers) assert response.status_code == 200 # now try with a new allowed origin, which should also be accepted headers = { **auth_header, - "Origin": payload["security"]["cors_origins"][0], + "Origin": "http://acme1.example.com", "Access-Control-Request-Method": "PATCH", } response = api_client.options(url, headers=headers) @@ -390,7 +420,7 @@ def test_patch_application_config_updates_cors_domains_rejects_invalid_urls( ) assert response.status_code == 422 - payload = {"security": {"cors_origins": ["http://test.com/"]}} + payload = {"security": {"cors_origins": ["http://test.com/path"]}} response = api_client.patch( url, headers=auth_header, @@ -703,7 +733,7 @@ def test_put_application_config_updates_cors_domains_in_middleware( current_cors_middleware = find_cors_middleware(api_client.app) - assert set(current_cors_middleware.options["allow_origins"]) == set( + assert set(current_cors_middleware.kwargs["allow_origins"]) == set( new_cors_origins ).union(set(original_cors_middleware_origins)) @@ -750,7 +780,7 @@ def test_put_application_config_updates_cors_domains_in_middleware( assert response.status_code == 200 current_cors_middleware = find_cors_middleware(api_client.app) - assert set(current_cors_middleware.options["allow_origins"]) == set( + assert set(current_cors_middleware.kwargs["allow_origins"]) == set( original_cors_middleware_origins ) # assert our cors middleware has been reset to original values @@ -1096,6 +1126,7 @@ def test_get_config( # allowlist additions should be made with care, and _need_ to be reviewed by the # Ethyca security team allowed_top_level_config_keys = { + "admin_ui", "user", "logging", "notifications", @@ -1216,3 +1247,8 @@ def test_get_config( assert ( len(consent_keys.difference(set(["override_vendor_purposes"]))) == 0 ), "Unexpected config API change, please review with Ethyca security team" + + execution_keys = set(config["admin_ui"].keys()) + assert ( + len(execution_keys.difference(set(["enabled", "url"]))) == 0 + ), "Unexpected config API change, please review with Ethyca security team" diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index e05b4ddd7e..b46b0eba77 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -383,7 +383,7 @@ def test_patch_connections_bulk_create_limit_exceeded( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "ensure this value has at most 50 items" + == "List should have at most 50 items after validation, not 51" ) @mock.patch("fides.api.util.connection_util.queue_privacy_request") @@ -525,6 +525,7 @@ def test_patch_connections_bulk_update( ) assert postgres_resource.access.value == "read" assert postgres_resource.disabled + assert postgres_resource.secrets == payload[0]["secrets"] mongo_connection = response_body["succeeded"][1] @@ -1420,10 +1421,9 @@ def test_put_connection_config_secrets_schema_validation( json=payload, ) assert resp.status_code == 422 - assert json.loads(resp.text)["detail"][0]["msg"] == "field required" assert ( - json.loads(resp.text)["detail"][1]["msg"] - == "PostgreSQLSchema must be supplied all of: ['host', 'dbname']." + resp.json()["detail"][0]["msg"] + == "Value error, PostgreSQLSchema must be supplied all of: ['host', 'dbname']." ) payload = { @@ -1438,8 +1438,14 @@ def test_put_connection_config_secrets_schema_validation( ) assert resp.status_code == 422 assert ( - json.loads(resp.text)["detail"][0]["msg"] == "value is not a valid integer" + json.loads(resp.text)["detail"][0]["msg"] + == "Input should be a valid integer, unable to parse string as an integer" ) + assert set(resp.json()["detail"][0].keys()) == { + "loc", + "msg", + "type", + } # Assert url, input, ctx keys have been removed to suppress sensitive information def test_put_connection_config_secrets( self, @@ -1648,6 +1654,21 @@ def test_put_connection_config_bigquery_secrets( assert bigquery_connection_config_without_secrets.last_test_timestamp is None assert bigquery_connection_config_without_secrets.last_test_succeeded is None + payload = { + "dataset": "some-dataset", + "keyfile_creds": "bad_creds", + } + resp = api_client.put( + url + "?verify=False", + headers=auth_header, + json=payload, + ) + assert set(resp.json()["detail"][0].keys()) == { + "type", + "loc", + "msg", + } # url, input, and ctx have been removed to help suppress sensitive information + def test_put_connection_config_snowflake_secrets( self, api_client: TestClient, diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index ba80e43a5b..8a35b04324 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -1,4 +1,3 @@ -from typing import List, Set from unittest import mock import pytest @@ -833,7 +832,10 @@ def test_get_connection_secret_schema_bigquery( "properties": { "type": {"title": "Type", "type": "string"}, "project_id": {"title": "Project ID", "type": "string"}, - "private_key_id": {"title": "Private Key ID", "type": "string"}, + "private_key_id": { + "title": "Private Key ID", + "type": "string", + }, "private_key": { "title": "Private Key", "sensitive": True, @@ -1259,7 +1261,6 @@ def test_get_connection_secret_schema_hubspot( assert resp.json() == { "title": "hubspot_schema", - "description": "Hubspot secrets schema", "type": "object", "properties": { "domain": { @@ -1294,6 +1295,64 @@ def test_get_connection_secrets_manual_webhook( "properties": {}, } + def test_get_connection_secrets_attentive( + self, api_client: TestClient, generate_auth_header, base_url + ): + auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ]) + resp = api_client.get( + base_url.format(connection_type="attentive"), headers=auth_header + ) + assert resp.status_code == 200 + + assert resp.json() == { + "additionalProperties": False, + "properties": { + "third_party_vendor_name": { + "default": "Attentive", + "title": "Third Party Vendor Name", + "type": "string", + }, + "recipient_email_address": { + "default": "privacy@attentive.com", + "format": "email", + "title": "Recipient Email Address", + "type": "string", + }, + "test_email_address": { + "title": "Test Email Address", + "format": "email", + "type": "string", + }, + "advanced_settings": { + "default": { + "identity_types": {"email": True, "phone_number": False} + }, + "allOf": [{"$ref": "#/definitions/AdvancedSettings"}], + }, + }, + "title": "AttentiveSchema", + "type": "object", + "definitions": { + "AdvancedSettings": { + "properties": { + "identity_types": {"$ref": "#/definitions/IdentityTypes"} + }, + "required": ["identity_types"], + "title": "AdvancedSettings", + "type": "object", + }, + "IdentityTypes": { + "properties": { + "email": {"title": "Email", "type": "boolean"}, + "phone_number": {"title": "Phone Number", "type": "boolean"}, + }, + "required": ["email", "phone_number"], + "title": "IdentityTypes", + "type": "object", + }, + }, + } + class TestInstantiateConnectionFromTemplate: @pytest.fixture(scope="function") @@ -1386,17 +1445,12 @@ def test_template_secrets_validation( ) assert resp.status_code == 422 - assert resp.json()["detail"][0] == { - "loc": ["domain"], - "msg": "field required", - "type": "value_error.missing", - } # extra values should be permitted, but the system should return an error if there are missing fields. - assert resp.json()["detail"][1] == { - "loc": ["__root__"], - "msg": "mailchimp_schema must be supplied all of: [domain, username, api_key].", - "type": "value_error", - } + assert ( + resp.json()["detail"][0]["msg"] + == "Value error, mailchimp_schema must be supplied all of: [domain, username, api_key]." + ) + assert resp.json()["detail"][0]["type"] == "value_error" connection_config = ConnectionConfig.filter( db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") @@ -1507,11 +1561,12 @@ def test_invalid_instance_key(self, db, generate_auth_header, api_client, base_u headers=auth_header, json=request_body, ) - assert resp.json()["detail"][0] == { - "loc": ["body", "instance_key"], - "msg": "FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: < this is an invalid key! >", - "type": "value_error.fidesvalidation", - } + assert resp.json()["detail"][0]["loc"] == ["body", "instance_key"] + assert ( + resp.json()["detail"][0]["msg"] + == "Value error, FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: < this is an invalid key! >" + ) + assert resp.json()["detail"][0]["type"] == "value_error" @mock.patch( "fides.api.api.v1.endpoints.saas_config_endpoints.upsert_dataset_config_from_template" diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py index 03fd1da6d3..aa8da70aca 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py @@ -253,7 +253,7 @@ def test_put_validate_dataset_invalid_length( assert response.status_code == 422 assert ( json.loads(response.text)["detail"][0]["msg"] - == "ensure this value is greater than 0" + == "Input should be greater than 0" ) def test_put_validate_dataset_invalid_data_type( @@ -293,7 +293,7 @@ def test_put_validate_dataset_invalid_data_type( assert response.status_code == 422 assert ( json.loads(response.text)["detail"][0]["msg"] - == "The data type stringsssssss is not supported." + == "Value error, The data type stringsssssss is not supported." ) def test_put_validate_dataset_invalid_fidesops_meta( @@ -537,7 +537,10 @@ def test_create_dataset_configs_by_ctl_dataset_key( assert ( succeeded["fides_key"] == "postgres_example_subscriptions_dataset" ), "Returns the fides_key of the ctl_dataset not the DatasetConfig" - assert succeeded["collections"] == Dataset.from_orm(ctl_dataset).collections + assert succeeded["collections"] == [ + coll.model_dump(mode="json") + for coll in Dataset.model_validate(ctl_dataset).collections + ] dataset_config.delete(db) @@ -569,7 +572,7 @@ def test_create_datasetconfigs_bad_data_category( assert dataset_config is None assert ( response.json()["detail"][0]["msg"] - == "The data category bad_category is not supported." + == "Value error, The data category bad_category is not supported." ) def test_create_datasets_configs_invalid_connection_key( @@ -614,7 +617,7 @@ def test_patch_dataset_configs_bulk_create_limit_exceeded( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "ensure this value has at most 50 items" + == "List should have at most 50 items after validation, not 51" ) def test_patch_create_dataset_configs_bulk_create( @@ -649,20 +652,24 @@ def test_patch_create_dataset_configs_bulk_create( db=db, field="fides_key", value="test_fides_key" ) assert first_dataset_config.ctl_dataset == ctl_dataset - assert ( - response_body["succeeded"][0]["collections"] - == Dataset.from_orm(first_dataset_config.ctl_dataset).collections - ) + assert response_body["succeeded"][0]["collections"] == [ + coll.model_dump(mode="json") + for coll in Dataset.model_validate( + first_dataset_config.ctl_dataset + ).collections + ] assert response_body["succeeded"][0]["fides_key"] == ctl_dataset.fides_key assert len(first_dataset_config.ctl_dataset.collections) == 1 second_dataset_config = DatasetConfig.get_by( db=db, field="fides_key", value="second_dataset_config" ) - assert ( - response_body["succeeded"][1]["collections"] - == Dataset.from_orm(second_dataset_config.ctl_dataset).collections - ) + assert response_body["succeeded"][1]["collections"] == [ + coll.model_dump(mode="json") + for coll in Dataset.model_validate( + second_dataset_config.ctl_dataset + ).collections + ] assert response_body["succeeded"][1]["fides_key"] == ctl_dataset.fides_key assert second_dataset_config.ctl_dataset == ctl_dataset @@ -987,7 +994,7 @@ def test_patch_datasets_bulk_create_limit_exceeded( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "ensure this value has at most 50 items" + == "List should have at most 50 items after validation, not 51" ) def test_patch_datasets_bulk_create( diff --git a/tests/ops/api/v1/endpoints/test_drp_endpoints.py b/tests/ops/api/v1/endpoints/test_drp_endpoints.py index b47fe1486f..6e65f175c1 100644 --- a/tests/ops/api/v1/endpoints/test_drp_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_drp_endpoints.py @@ -16,11 +16,7 @@ PrivacyRequestStatus, ) from fides.api.schemas.privacy_request import PrivacyRequestDRPStatus -from fides.api.util.cache import ( - cache_task_tracking_key, - get_drp_request_body_cache_key, - get_identity_cache_key, -) +from fides.api.util.cache import cache_task_tracking_key, get_drp_request_body_cache_key from fides.common.api.scope_registry import ( POLICY_READ, PRIVACY_REQUEST_READ, @@ -445,7 +441,9 @@ def test_get_privacy_request_with_drp_action( assert expected_drp_status.value == response.json()["status"] assert privacy_request_with_drp_action.id == response.json()["request_id"] assert ( - privacy_request_with_drp_action.requested_at.isoformat() + privacy_request_with_drp_action.requested_at.isoformat().replace( + "+00:00", "Z" + ) == response.json()["received_at"] ) diff --git a/tests/ops/api/v1/endpoints/test_manual_webhooks.py b/tests/ops/api/v1/endpoints/test_manual_webhooks.py index d964af80e4..5e027ba2b8 100644 --- a/tests/ops/api/v1/endpoints/test_manual_webhooks.py +++ b/tests/ops/api/v1/endpoints/test_manual_webhooks.py @@ -169,7 +169,7 @@ def test_access_manual_webhook_pii_field_too_long( assert response.status_code == 422 assert ( response.json()["detail"][0]["msg"] - == "ensure this value has at most 200 characters" + == "String should have at most 200 characters" ) def test_post_manual_webhook_duplicate_fields( @@ -184,7 +184,10 @@ def test_post_manual_webhook_duplicate_fields( auth_header = generate_auth_header([WEBHOOK_CREATE_OR_UPDATE]) response = api_client.post(url, headers=auth_header, json=payload) assert response.status_code == 422 - assert response.json()["detail"][0]["msg"] == "pii_fields must be unique" + assert ( + response.json()["detail"][0]["msg"] + == "Value error, pii_fields must be unique" + ) def test_post_access_manual_webhook_fields_empty_string( self, db, api_client, url, generate_auth_header @@ -200,7 +203,7 @@ def test_post_access_manual_webhook_fields_empty_string( assert response.status_code == 422 assert ( response.json()["detail"][0]["msg"] - == "ensure this value has at least 1 characters" + == "String should have at least 1 character" ) def test_post_access_manual_webhook_pii_label_spaces( @@ -217,7 +220,7 @@ def test_post_access_manual_webhook_pii_label_spaces( assert response.status_code == 422 assert ( response.json()["detail"][0]["msg"] - == "ensure this value has at least 1 characters" + == "String should have at least 1 character" ) def test_post_access_manual_webhook_dsr_package_labels_empty_string( @@ -309,7 +312,7 @@ def test_post_webhook_no_fields( assert response.status_code == 422 assert ( response.json()["detail"][0]["msg"] - == "ensure this value has at least 1 items" + == "List should have at least 1 item after validation, not 0" ) def test_post_manual_webhook( diff --git a/tests/ops/api/v1/endpoints/test_masking_endpoints.py b/tests/ops/api/v1/endpoints/test_masking_endpoints.py index ed094139e8..0858814d15 100644 --- a/tests/ops/api/v1/endpoints/test_masking_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_masking_endpoints.py @@ -30,11 +30,10 @@ def test_read_strategies(self, api_client: TestClient, generate_auth_header): auth_header = generate_auth_header(scopes=[MASKING_READ]) expected_response = [] for strategy in MaskingStrategy.get_strategies(): - expected_response.append(strategy.get_description()) + expected_response.append(strategy.get_description().model_dump(mode="json")) response = api_client.get(V1_URL_PREFIX + MASKING_STRATEGY, headers=auth_header) - response_body = json.loads(response.text) - print(response_body) + response_body = response.json() assert 200 == response.status_code assert expected_response == response_body @@ -92,7 +91,7 @@ def test_mask_value_string_rewrite( ) assert 200 == response.status_code - assert expected_response == json.loads(response.text) + assert expected_response.model_dump(mode="json") == response.json() def test_mask_value_random_string_rewrite( self, api_client: TestClient, generate_auth_header diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index ac9b1cb713..97c6fd622c 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -24,7 +24,6 @@ MessagingServiceSecrets, MessagingServiceType, MessagingTemplateDefault, - MessagingTemplateWithPropertiesDetail, MessagingTemplateWithPropertiesSummary, ) from fides.common.api.scope_registry import ( @@ -133,8 +132,8 @@ def test_post_email_config_with_invalid_mailgun_details( auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.post(url, headers=auth_header, json=payload) assert 422 == response.status_code - assert response.json()["detail"][0]["msg"] == "field required" - assert response.json()["detail"][1]["msg"] == "extra fields not permitted" + assert response.json()["detail"][0]["msg"] == "Field required" + assert response.json()["detail"][1]["msg"] == "Extra inputs are not permitted" def test_post_mailgun_email_config_with_a_twilio_detail( self, @@ -150,8 +149,8 @@ def test_post_mailgun_email_config_with_a_twilio_detail( auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.post(url, headers=auth_header, json=payload) assert 422 == response.status_code - assert response.json()["detail"][0]["msg"] == "field required" - assert response.json()["detail"][1]["msg"] == "extra fields not permitted" + assert response.json()["detail"][0]["msg"] == "Field required" + assert response.json()["detail"][1]["msg"] == "Extra inputs are not permitted" def test_post_email_config_with_not_supported_service_type( self, @@ -168,7 +167,7 @@ def test_post_email_config_with_not_supported_service_type( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "value is not a valid enumeration member; permitted: 'mailgun', 'twilio_text', 'twilio_email', 'mailchimp_transactional'" + == "Input should be 'mailgun', 'twilio_text', 'twilio_email' or 'mailchimp_transactional'" ) def test_post_email_config_with_no_key( @@ -202,8 +201,8 @@ def test_post_email_config_with_invalid_key( response = api_client.post(url, headers=auth_header, json=payload) assert 422 == response.status_code assert ( - json.loads(response.text)["detail"][0]["msg"] - == "FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: *invalid-key" + response.json()["detail"][0]["msg"] + == "Value error, FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: *invalid-key" ) def test_post_email_config_with_key( @@ -256,7 +255,7 @@ def test_post_email_config_missing_detail( ) assert response.status_code == 422 errors = response.json()["detail"] - assert errors[0]["msg"] == "Messaging config must include details" + assert errors[0]["msg"] == "Value error, Messaging config must include details" def test_post_email_config_service_already_exists( self, @@ -527,13 +526,15 @@ def test_update_with_invalid_secrets_key( auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json={"bad_key": "12345"}) - assert response.status_code == 400 - assert response.json() == { - "detail": [ - "field required ('mailgun_api_key',)", - "extra fields not permitted ('bad_key',)", - ] - } + assert response.status_code == 422 + assert response.json()["detail"] == [ + {"type": "missing", "loc": ["mailgun_api_key"], "msg": "Field required"}, + { + "type": "extra_forbidden", + "loc": ["bad_key"], + "msg": "Extra inputs are not permitted", + }, + ] def test_put_config_secrets( self, @@ -755,10 +756,13 @@ def test_put_config_secrets_with_sender_phone_incorrect_format( } auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json=payload) - assert response.status_code == 400 + + # Because validation failed on all members, all errors were returned. This was + # just one of them + assert response.status_code == 422 assert ( - f"Phone number must be formatted in E.164 format, i.e. '+15558675309'. ('twilio_sender_phone_number',)" - in response.json()["detail"] + f"Phone number must be formatted in E.164 format, i.e. '+15558675309'" + in response.text ) def test_put_config_secrets_with_no_sender_phone_nor_messaging_service_id( @@ -775,10 +779,10 @@ def test_put_config_secrets_with_no_sender_phone_nor_messaging_service_id( } auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json=payload) - assert response.status_code == 400 + assert response.status_code == 422 assert ( - f"Either the twilio_messaging_service_sid or the twilio_sender_phone_number should be supplied. ('__root__',)" - in response.json()["detail"] + f"Either the twilio_messaging_service_sid or the twilio_sender_phone_number should be supplied" + in response.text ) @@ -1299,9 +1303,9 @@ def test_update_default_with_invalid_secrets_key( ): auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json={"bad_key": "12345"}) - assert response.status_code == 400 - assert "field required" in response.text - assert "extra fields not permitted" in response.text + assert response.status_code == 422 + assert "Field required" in response.text + assert "Extra inputs are not permitted" in response.text @mock.patch("fides.api.models.messaging.MessagingConfig.set_secrets") def test_update_default_set_secrets_error( diff --git a/tests/ops/api/v1/endpoints/test_policy_endpoints.py b/tests/ops/api/v1/endpoints/test_policy_endpoints.py index c7b0c879a3..e0eaff4bd6 100644 --- a/tests/ops/api/v1/endpoints/test_policy_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_policy_endpoints.py @@ -588,7 +588,7 @@ def test_create_polices_limit_exceeded( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "ensure this value has at most 50 items" + == "List should have at most 50 items after validation, not 51" ) def test_create_multiple_policies( @@ -750,7 +750,7 @@ def test_create_policy_invalid_drp_action( response_body = json.loads(resp.text) assert ( - "value is not a valid enumeration member; permitted: 'access', 'deletion', 'sale:opt_out', 'sale:opt_in', 'access:categories', 'access:specific'" + "Input should be 'access', 'deletion', 'sale:opt_out', 'sale:opt_in', 'access:categories' or 'access:specific'" == response_body["detail"][0]["msg"] ) @@ -882,7 +882,7 @@ def test_create_policy_with_invalid_key( assert resp.status_code == 422 assert ( json.loads(resp.text)["detail"][0]["msg"] - == "FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: here*is*an*invalid*key" + == "Value error, FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: here*is*an*invalid*key" ) def test_create_policy_already_exists( @@ -1009,7 +1009,7 @@ def test_create_rules_limit_exceeded( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "ensure this value has at most 50 items" + == "List should have at most 50 items after validation, not 51" ) def test_create_access_rule_for_policy( @@ -1426,7 +1426,7 @@ def test_create_targets_limit_exceeded( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "ensure this value has at most 50 items" + == "List should have at most 50 items after validation, not 51" ) def test_update_rule_targets( diff --git a/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py b/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py index db59502d14..ca236a19e3 100644 --- a/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py @@ -1,4 +1,5 @@ import json +from copy import copy from typing import Dict import pytest @@ -24,13 +25,16 @@ def embedded_http_connection_config(connection_config: ConnectionConfig) -> Dict: """Helper to reduce clutter - a lot of the tests below assert the entire response body, which includes the https connection config""" + created_at = copy(connection_config.created_at) + updated_at = copy(connection_config.updated_at) + return { "name": connection_config.name, "key": connection_config.key, "connection_type": "https", "access": connection_config.access.value, - "created_at": stringify_date(connection_config.created_at), - "updated_at": stringify_date(connection_config.updated_at), + "created_at": stringify_date(created_at), + "updated_at": stringify_date(updated_at), "last_test_timestamp": None, "last_test_succeeded": None, "disabled": False, @@ -84,24 +88,22 @@ def test_get_pre_execution_policy_webhooks( assert resp.status_code == 200 body = json.loads(resp.text) + embedded_config = embedded_http_connection_config(https_connection_config) + assert body == { "items": [ { "direction": "one_way", "key": "pre_execution_one_way_webhook", "name": policy_pre_execution_webhooks[0].name, - "connection_config": embedded_http_connection_config( - https_connection_config - ), + "connection_config": embedded_config, "order": 0, }, { "direction": "two_way", "key": "pre_execution_two_way_webhook", "name": policy_pre_execution_webhooks[1].name, - "connection_config": embedded_http_connection_config( - https_connection_config - ), + "connection_config": embedded_config, "order": 1, }, ], @@ -434,10 +436,7 @@ def test_direction_error_fails_all( ) assert resp.status_code == 422 body = json.loads(resp.text) - assert ( - body["detail"][0]["msg"] - == "value is not a valid enumeration member; permitted: 'one_way', 'two_way'" - ) + assert body["detail"][0]["msg"] == "Input should be 'one_way' or 'two_way'" assert db.query(PolicyPreWebhook).count() == 0 # All must succeed or fail def test_put_pre_execution_webhooks_duplicate_keys( diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 741c51e3e1..77c46feb10 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -100,7 +100,7 @@ def stringify_date(log_date: datetime) -> str: - return log_date.strftime("%Y-%m-%dT%H:%M:%S.%f+00:00") + return log_date.strftime("%Y-%m-%dT%H:%M:%S.%fZ") class TestCreatePrivacyRequest: @@ -491,7 +491,7 @@ def test_create_privacy_request_limit_exceeded( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "ensure this value has at most 50 items" + == "List should have at most 50 items after validation, not 51" ) @mock.patch( @@ -624,7 +624,10 @@ def test_create_privacy_request_invalid_encryption_values( ] resp = api_client.post(url, json=data) assert resp.status_code == 422 - assert resp.json()["detail"][0]["msg"] == "Encryption key must be 16 bytes long" + assert ( + resp.json()["detail"][0]["msg"] + == "Value error, Encryption key must be 16 bytes long" + ) @mock.patch( "fides.api.service.privacy_request.request_runner_service.run_privacy_request.delay" @@ -937,8 +940,8 @@ def test_get_privacy_requests_by_id( "name": privacy_request.policy.name, "key": privacy_request.policy.key, "rules": [ - rule.dict() - for rule in PolicyResponse.from_orm( + rule.model_dump(mode="json") + for rule in PolicyResponse.model_validate( privacy_request.policy ).rules ], @@ -1001,8 +1004,8 @@ def test_get_privacy_requests_by_partial_id( "name": privacy_request.policy.name, "key": privacy_request.policy.key, "rules": [ - rule.dict() - for rule in PolicyResponse.from_orm( + rule.model_dump(mode="json") + for rule in PolicyResponse.model_validate( privacy_request.policy ).rules ], @@ -1432,8 +1435,8 @@ def test_verbose_privacy_requests( "name": privacy_request.policy.name, "key": privacy_request.policy.key, "rules": [ - rule.dict() - for rule in PolicyResponse.from_orm( + rule.model_dump(mode="json") + for rule in PolicyResponse.model_validate( privacy_request.policy ).rules ], @@ -1948,8 +1951,8 @@ def test_privacy_request_search_by_id( "name": privacy_request.policy.name, "key": privacy_request.policy.key, "rules": [ - rule.dict() - for rule in PolicyResponse.from_orm( + rule.model_dump(mode="json") + for rule in PolicyResponse.model_validate( privacy_request.policy ).rules ], @@ -2012,8 +2015,8 @@ def test_privacy_request_search_by_partial_id( "name": privacy_request.policy.name, "key": privacy_request.policy.key, "rules": [ - rule.dict() - for rule in PolicyResponse.from_orm( + rule.model_dump(mode="json") + for rule in PolicyResponse.model_validate( privacy_request.policy ).rules ], @@ -2522,8 +2525,8 @@ def test_verbose_privacy_requests( "name": privacy_request.policy.name, "key": privacy_request.policy.key, "rules": [ - rule.dict() - for rule in PolicyResponse.from_orm( + rule.model_dump(mode="json") + for rule in PolicyResponse.model_validate( privacy_request.policy ).rules ], @@ -3517,7 +3520,9 @@ def test_approve_privacy_request_creates_audit_log_and_sends_email( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["to_identity"] == Identity( + email="test@example.com" + ).model_dump(mode="json") assert task_kwargs["service_type"] == MessagingServiceType.mailgun.value message_meta = task_kwargs["message_meta"] @@ -4142,7 +4147,9 @@ def test_deny_privacy_request_without_denial_reason( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["to_identity"] == Identity( + email="test@example.com" + ).model_dump(mode="json") assert task_kwargs["service_type"] == MessagingServiceType.mailgun.value message_meta = task_kwargs["message_meta"] @@ -4152,7 +4159,7 @@ def test_deny_privacy_request_without_denial_reason( ) assert message_meta["body_params"] == RequestReviewDenyBodyParams( rejection_reason=None - ) + ).model_dump(mode="json") queue = call_args["queue"] assert queue == MESSAGING_QUEUE_NAME @@ -4213,7 +4220,9 @@ def test_deny_privacy_request_with_denial_reason( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["to_identity"] == Identity( + email="test@example.com" + ).model_dump(mode="json") assert task_kwargs["service_type"] == MessagingServiceType.mailgun.value message_meta = task_kwargs["message_meta"] @@ -4223,7 +4232,7 @@ def test_deny_privacy_request_with_denial_reason( ) assert message_meta["body_params"] == RequestReviewDenyBodyParams( rejection_reason=denial_reason - ) + ).model_dump(mode="json") queue = call_args["queue"] assert queue == MESSAGING_QUEUE_NAME @@ -4388,8 +4397,10 @@ def test_resume_privacy_request( "key": privacy_request.policy.key, "name": privacy_request.policy.name, "rules": [ - rule.dict() - for rule in PolicyResponse.from_orm(privacy_request.policy).rules + rule.model_dump(mode="json") + for rule in PolicyResponse.model_validate( + privacy_request.policy + ).rules ], }, "action_required_details": None, @@ -4889,7 +4900,7 @@ def test_verify_identity_no_admin_approval_needed( task_kwargs = call_args["kwargs"] assert task_kwargs["to_identity"] == Identity( phone_number="+12345678910", email="test@example.com" - ) + ).model_dump(mode="json") assert task_kwargs["service_type"] == MessagingServiceType.mailgun.value message_meta = task_kwargs["message_meta"] @@ -4898,7 +4909,7 @@ def test_verify_identity_no_admin_approval_needed( ) assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} - ) + ).model_dump(mode="json") queue = call_args["queue"] assert queue == MESSAGING_QUEUE_NAME @@ -5051,7 +5062,7 @@ def test_verify_identity_admin_approval_needed( task_kwargs = call_args["kwargs"] assert task_kwargs["to_identity"] == Identity( phone_number="+12345678910", email="test@example.com" - ) + ).model_dump(mode="json") assert task_kwargs["service_type"] == MessagingServiceType.mailgun.value message_meta = task_kwargs["message_meta"] @@ -5060,7 +5071,7 @@ def test_verify_identity_admin_approval_needed( ) assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} - ) + ).model_dump(mode="json") queue = call_args["queue"] assert queue == MESSAGING_QUEUE_NAME @@ -5257,7 +5268,7 @@ def test_supply_invalid_fields( url, headers=auth_header, json={"bad_field": "value"} ) assert 422 == response.status_code - assert response.json()["detail"][0]["msg"] == "extra fields not permitted" + assert response.json()["detail"][0]["msg"] == "Extra inputs are not permitted" def test_patch_inputs_bad_privacy_request_status( self, @@ -5407,7 +5418,7 @@ def test_supply_invalid_fields( url, headers=auth_header, json={"bad_field": "value"} ) assert 422 == response.status_code - assert response.json()["detail"][0]["msg"] == "extra fields not permitted" + assert response.json()["detail"][0]["msg"] == "Extra inputs are not permitted" def test_patch_inputs_bad_privacy_request_status( self, @@ -6008,7 +6019,9 @@ def test_create_privacy_request_no_email_config( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["to_identity"] == Identity( + email="test@example.com" + ).model_dump(mode="json") assert task_kwargs["service_type"] == MessagingServiceType.mailgun.value message_meta = task_kwargs["message_meta"] @@ -6017,7 +6030,7 @@ def test_create_privacy_request_no_email_config( ) assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} - ) + ).model_dump(mode="json") queue = call_args["queue"] assert queue == MESSAGING_QUEUE_NAME @@ -6059,7 +6072,9 @@ def test_create_privacy_request_with_email_config( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["to_identity"] == Identity( + email="test@example.com" + ).model_dump(mode="json") assert task_kwargs["service_type"] == MessagingServiceType.mailgun.value message_meta = task_kwargs["message_meta"] @@ -6068,7 +6083,7 @@ def test_create_privacy_request_with_email_config( ) assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} - ) + ).model_dump(mode="json") queue = call_args["queue"] assert queue == MESSAGING_QUEUE_NAME @@ -6280,7 +6295,7 @@ def test_create_privacy_request_limit_exceeded( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "ensure this value has at most 50 items" + == "List should have at most 50 items after validation, not 51" ) @mock.patch( @@ -6416,7 +6431,10 @@ def test_create_privacy_request_invalid_encryption_values( auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) resp = api_client.post(url, json=data, headers=auth_header) assert resp.status_code == 422 - assert resp.json()["detail"][0]["msg"] == "Encryption key must be 16 bytes long" + assert ( + resp.json()["detail"][0]["msg"] + == "Value error, Encryption key must be 16 bytes long" + ) @mock.patch( "fides.api.service.privacy_request.request_runner_service.run_privacy_request.delay" diff --git a/tests/ops/api/v1/endpoints/test_registration_endpoints.py b/tests/ops/api/v1/endpoints/test_registration_endpoints.py index a7ee947d67..5b44aca5b1 100644 --- a/tests/ops/api/v1/endpoints/test_registration_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_registration_endpoints.py @@ -6,6 +6,26 @@ from fides.common.api.v1.urn_registry import REGISTRATION, V1_URL_PREFIX +class TestUserRegistrationModel: + + def test_registration_as_log(self, db): + EXAMPLE_ANALYTICS_ID = "example-analytics-id" + OPT_IN = True + + user_reg = UserRegistration.create( + db, + data={ + "user_email": "user@example.com", + "user_organization": "Example Org.", + "analytics_id": EXAMPLE_ANALYTICS_ID, + "opt_in": OPT_IN, + }, + ) + + fideslog_reg = user_reg.as_fideslog() + assert fideslog_reg.organization == "Example Org." + + class TestUserRegistration: """Tests for the UserRegistration API, configured during `fides deploy`.""" diff --git a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py index 6b509af753..b85393cf81 100644 --- a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py @@ -124,7 +124,7 @@ def test_put_validate_saas_config_reference_and_identity( details = json.loads(response.text)["detail"] assert ( details[0]["msg"] - == "Must have exactly one of 'identity', 'references', or 'connector_param'" + == "Value error, Must have exactly one of 'identity', 'references', or 'connector_param'" ) def test_put_validate_saas_config_wrong_reference_direction( @@ -153,7 +153,7 @@ def test_put_validate_saas_config_wrong_reference_direction( details = json.loads(response.text)["detail"] assert ( details[0]["msg"] - == "References can only have a direction of 'from', found 'to'" + == "Value error, References can only have a direction of 'from', found 'to'" ) @@ -671,7 +671,7 @@ def test_register_connector_template_wrong_scope( "connector_template_invalid_config", 400, { - "detail": "1 validation error for SaaSConfig\ntest_request\n field required (type=value_error.missing)" + "detail": "1 validation error for SaaSConfig\ntest_request\n Field required [type=missing, input_value={'fides_key': ' 0 -> name\n field required (type=value_error.missing)" + "detail": "1 validation error for Dataset\ncollections.0.name\n Field required [type=missing, input_value={'fides_meta': None, 'nam...': ['user.unique_id']}]}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.7/v/missing" }, ), ( diff --git a/tests/ops/api/v1/endpoints/test_storage_endpoints.py b/tests/ops/api/v1/endpoints/test_storage_endpoints.py index 292b39012d..4c94b4bc18 100644 --- a/tests/ops/api/v1/endpoints/test_storage_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_storage_endpoints.py @@ -108,7 +108,9 @@ def test_post_upload_data( ) call_kwargs = mock_post_upload_data.call_args.kwargs assert call_kwargs["privacy_request"].id == privacy_request.id - assert response_body == DataUpload(location=expected_location) + assert response_body == DataUpload(location=expected_location).model_dump( + mode="json" + ) class TestPatchStorageConfig: @@ -184,7 +186,7 @@ def test_put_storage_config_with_invalid_key( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: *invalid-key" + == "Value error, FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: *invalid-key" ) def test_patch_storage_config_with_key( @@ -318,7 +320,7 @@ def test_patch_storage_config_missing_detail( assert response.status_code == 422 errors = response.json()["detail"] assert "details" in errors[0]["loc"] - assert errors[0]["msg"] == "[\"field required ('bucket',)\"]" + assert errors[0]["msg"] == "Value error, [\"Field required ('bucket',)\"]" class TestPutStorageConfigSecretsS3: @@ -362,13 +364,11 @@ def test_update_with_invalid_secrets_key( url + "?verify=False", headers=auth_header, json={"bad_key": "12345"} ) assert response.status_code == 400 - assert response.json() == { - "detail": [ - "field required ('aws_access_key_id',)", - "field required ('aws_secret_access_key',)", - "extra fields not permitted ('bad_key',)", - ] - } + assert response.json()["detail"] == [ + "Field required ('aws_access_key_id',)", + "Field required ('aws_secret_access_key',)", + "Extra inputs are not permitted ('bad_key',)", + ] def test_put_config_secrets_without_verifying( self, @@ -1009,7 +1009,7 @@ def test_put_default_storage_config_missing_detail( }, ) assert response.status_code == 422 - assert "field required" in response.text + assert "Field required" in response.text assert "bucket" in response.text @mock.patch("fides.api.models.storage.StorageConfig.create_or_update") @@ -1108,14 +1108,13 @@ def test_update_default_with_invalid_secrets_key( response = api_client.put( url + "?verify=False", headers=auth_header, json={"bad_key": "12345"} ) + + assert response.json()["detail"] == [ + "Field required ('aws_access_key_id',)", + "Field required ('aws_secret_access_key',)", + "Extra inputs are not permitted ('bad_key',)", + ] assert response.status_code == 400 - assert response.json() == { - "detail": [ - "field required ('aws_access_key_id',)", - "field required ('aws_secret_access_key',)", - "extra fields not permitted ('bad_key',)", - ] - } @mock.patch("fides.api.models.storage.StorageConfig.set_secrets") def test_update_default_set_secrets_error( diff --git a/tests/ops/api/v1/endpoints/test_system.py b/tests/ops/api/v1/endpoints/test_system.py index 1519d92738..9927095212 100644 --- a/tests/ops/api/v1/endpoints/test_system.py +++ b/tests/ops/api/v1/endpoints/test_system.py @@ -731,17 +731,11 @@ def test_template_secrets_validation( ) assert resp.status_code == 422 - assert resp.json()["detail"][0] == { - "loc": ["domain"], - "msg": "field required", - "type": "value_error.missing", - } + assert ( + resp.json()["detail"][0]["msg"] + == "Value error, mailchimp_schema must be supplied all of: [domain, username, api_key]." + ) # extra values should be permitted, but the system should return an error if there are missing fields. - assert resp.json()["detail"][1] == { - "loc": ["__root__"], - "msg": "mailchimp_schema must be supplied all of: [domain, username, api_key].", - "type": "value_error", - } connection_config = ConnectionConfig.filter( db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") @@ -851,8 +845,8 @@ def test_invalid_instance_key(self, db, generate_auth_header, api_client, base_u ) assert resp.json()["detail"][0] == { "loc": ["body", "instance_key"], - "msg": "FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: < this is an invalid key! >", - "type": "value_error.fidesvalidation", + "msg": "Value error, FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'. Value provided: < this is an invalid key! >", + "type": "value_error", } @mock.patch( diff --git a/tests/ops/api/v1/endpoints/test_user_endpoints.py b/tests/ops/api/v1/endpoints/test_user_endpoints.py index 0833b09ad9..a0517fcba6 100644 --- a/tests/ops/api/v1/endpoints/test_user_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_user_endpoints.py @@ -1,5 +1,5 @@ import json -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from unittest import mock from uuid import uuid4 @@ -55,6 +55,7 @@ ) from fides.config import CONFIG from tests.conftest import generate_auth_header_for_user +from tests.ops.api.v1.endpoints.test_privacy_request_endpoints import stringify_date page_size = Params().size @@ -128,7 +129,7 @@ def test_create_user_bad_password( assert HTTP_422_UNPROCESSABLE_ENTITY == response.status_code assert ( response.json()["detail"][0]["msg"] - == "Password must have at least eight characters." + == "Value error, Password must have at least eight characters." ) body = { @@ -140,7 +141,7 @@ def test_create_user_bad_password( assert HTTP_422_UNPROCESSABLE_ENTITY == response.status_code assert ( response.json()["detail"][0]["msg"] - == "Password must have at least one number." + == "Value error, Password must have at least one number." ) body = { @@ -152,7 +153,7 @@ def test_create_user_bad_password( assert HTTP_422_UNPROCESSABLE_ENTITY == response.status_code assert ( response.json()["detail"][0]["msg"] - == "Password must have at least one capital letter." + == "Value error, Password must have at least one capital letter." ) body = { @@ -165,9 +166,16 @@ def test_create_user_bad_password( assert HTTP_422_UNPROCESSABLE_ENTITY == response.status_code assert ( response.json()["detail"][0]["msg"] - == "Password must have at least one symbol." + == "Value error, Password must have at least one symbol." ) + # Tests request_validation_exception_handler which removes the user input + # and the pydantic url from the response. We don't want to reflect a password + # back in the response + assert "input" not in response.json()["detail"][0] + assert "url" not in response.json()["detail"][0] + assert "ctx" not in response.json()["detail"][0] + def test_create_user_no_email( self, api_client, @@ -714,7 +722,7 @@ def test_get_user( user_data = resp.json() assert user_data["username"] == application_user.username assert user_data["id"] == application_user.id - assert user_data["created_at"] == application_user.created_at.isoformat() + assert user_data["created_at"] == stringify_date(application_user.created_at) assert user_data["first_name"] == application_user.first_name assert user_data["last_name"] == application_user.last_name @@ -750,7 +758,7 @@ def test_update_different_users_names( user_data = resp.json() assert user_data["username"] == user.username assert user_data["id"] == user.id - assert user_data["created_at"] == user.created_at.isoformat() + assert user_data["created_at"] == stringify_date(user.created_at) assert user_data["first_name"] == NEW_FIRST_NAME assert user_data["last_name"] == NEW_LAST_NAME @@ -779,7 +787,7 @@ def test_update_user_names( user_data = resp.json() assert user_data["username"] == application_user.username assert user_data["id"] == application_user.id - assert user_data["created_at"] == application_user.created_at.isoformat() + assert user_data["created_at"] == stringify_date(application_user.created_at) assert user_data["first_name"] == NEW_FIRST_NAME assert user_data["last_name"] == NEW_LAST_NAME @@ -825,7 +833,7 @@ def test_user_can_update_their_own_name_without_scope( user_data = resp.json() assert user_data["username"] == application_user.username assert user_data["id"] == application_user.id - assert user_data["created_at"] == application_user.created_at.isoformat() + assert user_data["created_at"] == stringify_date(application_user.created_at) assert user_data["first_name"] == NEW_FIRST_NAME assert user_data["last_name"] == NEW_LAST_NAME diff --git a/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py b/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py index 31afa30e6e..7b1b0955e5 100644 --- a/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py @@ -132,7 +132,7 @@ def test_create_user_permissions_add_bad_role( response_body = response.json() assert HTTP_422_UNPROCESSABLE_ENTITY == response.status_code assert ( - "value is not a valid enumeration member" + "Input should be 'owner', 'viewer_and_approver', 'viewer', 'approver' or 'contributor'" in response_body["detail"][0]["msg"] ) diff --git a/tests/ops/api/v1/test_main.py b/tests/ops/api/v1/test_main.py index 9b4ef8c712..2c06e4d0df 100644 --- a/tests/ops/api/v1/test_main.py +++ b/tests/ops/api/v1/test_main.py @@ -4,7 +4,7 @@ from fastapi import FastAPI from starlette.testclient import TestClient -from fides.api.main import create_fides_app +from fides.api.main import create_fides_app, lifespan from fides.api.oauth.utils import ( get_root_client, verify_oauth_client, @@ -26,7 +26,7 @@ def test_configure_security_env_overrides_dev(self) -> None: """ security_env = "dev" test_app = FastAPI(title="test") - test_app = create_fides_app(security_env=security_env) + test_app = create_fides_app(lifespan=lifespan, security_env=security_env) assert ( test_app.dependency_overrides[verify_oauth_client_prod] == get_root_client ) @@ -38,7 +38,7 @@ def test_configure_security_env_overrides_prod(self) -> None: """ security_env = "prod" test_app = FastAPI(title="test") - test_app = create_fides_app(security_env=security_env) + test_app = create_fides_app(lifespan=lifespan, security_env=security_env) with pytest.raises(KeyError): test_app.dependency_overrides[verify_oauth_client] diff --git a/tests/ops/api/v1/test_middleware.py b/tests/ops/api/v1/test_middleware.py index 22f6ee010f..3b13e9bb31 100644 --- a/tests/ops/api/v1/test_middleware.py +++ b/tests/ops/api/v1/test_middleware.py @@ -18,8 +18,6 @@ from fides.common.api.scope_registry import USER_CREATE from fides.config import CONFIG -# from tests.conftest import generate_auth_header_for_user - page_size = Params().size import json diff --git a/tests/ops/generator/test_data_generator.py b/tests/ops/generator/test_data_generator.py index 725fa45b81..af9ab1cc62 100644 --- a/tests/ops/generator/test_data_generator.py +++ b/tests/ops/generator/test_data_generator.py @@ -48,7 +48,7 @@ def parse_yaml() -> GraphDataset: """Test that 'after' parameters are properly read""" d = yaml.safe_load(f) dataset = d.get("dataset")[0] - d: Dataset = Dataset.parse_obj(dataset) + d: Dataset = Dataset.model_validate(dataset) return convert_dataset_to_graph(d, "ignore") diff --git a/tests/ops/graph/test_config.py b/tests/ops/graph/test_config.py index 042c1c38b1..d16c0da48a 100644 --- a/tests/ops/graph/test_config.py +++ b/tests/ops/graph/test_config.py @@ -356,7 +356,9 @@ def test_field_paths_by_category(self): } def test_collection_json(self): - json_collection = json.loads(collection_to_serialize.json()) + json_collection = json.loads( + collection_to_serialize.model_dump_json(serialize_as_any=True) + ) assert json_collection == serialized_collection def test_parse_from_task(self): @@ -461,7 +463,7 @@ def test_field_data_type(self): assert field.data_type() == "None" field = ObjectField( - name="integer test", data_type_converter=ObjectTypeConverter(), fields=[] + name="integer test", data_type_converter=ObjectTypeConverter(), fields={} ) assert field.data_type() == "object" @@ -529,7 +531,7 @@ def test_generate_object_field_with_data_categories(self): data_categories=["user.contact.address.street"], ) - with pytest.raises(pydantic.error_wrappers.ValidationError): + with pytest.raises(pydantic.ValidationError): generate_field( name="obj", data_categories=["A.B.C"], diff --git a/tests/ops/integration_tests/setup_scripts/mariadb_setup.py b/tests/ops/integration_tests/setup_scripts/mariadb_setup.py index 873c31f65a..729ca80713 100644 --- a/tests/ops/integration_tests/setup_scripts/mariadb_setup.py +++ b/tests/ops/integration_tests/setup_scripts/mariadb_setup.py @@ -10,6 +10,7 @@ ConnectionConfig, ConnectionType, ) +from fides.api.models.privacy_experience import PrivacyExperienceConfig from fides.api.service.connectors.sql_connector import MariaDBConnector from fides.config import CONFIG diff --git a/tests/ops/integration_tests/setup_scripts/mssql_setup.py b/tests/ops/integration_tests/setup_scripts/mssql_setup.py index 97c4b7223e..0837330777 100644 --- a/tests/ops/integration_tests/setup_scripts/mssql_setup.py +++ b/tests/ops/integration_tests/setup_scripts/mssql_setup.py @@ -4,6 +4,8 @@ import sqlalchemy from sqlalchemy.exc import SQLAlchemyError +from fides.api.models.privacy_experience import PrivacyExperienceConfig + MSSQL_URL_TEMPLATE = "mssql+pymssql://sa:Mssql_pw1@mssql_example:1433/{}" MASTER_MSSQL_URL = MSSQL_URL_TEMPLATE.format("master") diff --git a/tests/ops/integration_tests/setup_scripts/timescale_setup.py b/tests/ops/integration_tests/setup_scripts/timescale_setup.py index 24dfbe444a..4da3bb0dbd 100644 --- a/tests/ops/integration_tests/setup_scripts/timescale_setup.py +++ b/tests/ops/integration_tests/setup_scripts/timescale_setup.py @@ -12,6 +12,7 @@ ConnectionConfig, ConnectionType, ) +from fides.api.models.privacy_experience import PrivacyExperienceConfig from fides.api.service.connectors import TimescaleConnector from fides.config import CONFIG diff --git a/tests/ops/integration_tests/test_connection_configuration_integration.py b/tests/ops/integration_tests/test_connection_configuration_integration.py index 6ec368a2a9..e938c8543b 100644 --- a/tests/ops/integration_tests/test_connection_configuration_integration.py +++ b/tests/ops/integration_tests/test_connection_configuration_integration.py @@ -17,7 +17,6 @@ get_connector, ) from fides.api.service.connectors.sql_connector import ( - GoogleCloudSQLMySQLConnector, MariaDBConnector, MicrosoftSQLServerConnector, MySQLConnector, @@ -1187,7 +1186,10 @@ def test_saas_connection_connect_missing_secrets( assert resp.status_code == 422 body = json.loads(resp.text) - assert body["detail"][0]["msg"] == "field required" + assert ( + body["detail"][0]["msg"] + == "Value error, mailchimp_schema must be supplied all of: [domain, username, api_key]." + ) def test_saas_connection_connect_with_extra_secrets( self, diff --git a/tests/ops/integration_tests/test_enabled_actions.py b/tests/ops/integration_tests/test_enabled_actions.py index 1276ed7556..de455669af 100644 --- a/tests/ops/integration_tests/test_enabled_actions.py +++ b/tests/ops/integration_tests/test_enabled_actions.py @@ -26,7 +26,9 @@ def dataset_graph( integration_postgres_config, ) -> DatasetGraph: dataset_postgres = Dataset(**example_datasets[0]) - dataset_config(db, integration_postgres_config, dataset_postgres.dict()) + dataset_config( + db, integration_postgres_config, dataset_postgres.model_dump(mode="json") + ) graph = convert_dataset_to_graph( dataset_postgres, integration_postgres_config.key ) diff --git a/tests/ops/integration_tests/test_execution.py b/tests/ops/integration_tests/test_execution.py index 45ae271c3c..80f69e1165 100644 --- a/tests/ops/integration_tests/test_execution.py +++ b/tests/ops/integration_tests/test_execution.py @@ -904,6 +904,7 @@ async def test_restart_graph_from_failure( {"email": "customer-4@example.com"}, db, ) + assert exc.value.__class__ == ValidationError assert ( "MongoDBSchema must be supplied all of: ['host', 'username', 'password', 'defaultauthdb']" diff --git a/tests/ops/integration_tests/test_external_database_connections.py b/tests/ops/integration_tests/test_external_database_connections.py index a5dc70b665..ca3b65c3cc 100644 --- a/tests/ops/integration_tests/test_external_database_connections.py +++ b/tests/ops/integration_tests/test_external_database_connections.py @@ -54,7 +54,7 @@ def redshift_test_engine() -> Generator: database=database, db_schema=db_schema, ) - connection_config.secrets = schema.dict() + connection_config.secrets = schema.model_dump(mode="json") connector: RedshiftConnector = get_connector(connection_config) engine = connector.client() @@ -98,7 +98,7 @@ def snowflake_test_engine() -> Generator: name="My Snowflake Config", key="test_snowflake_key", connection_type=ConnectionType.snowflake, - secrets=schema.dict(), + secrets=schema.model_dump(mode="json"), ) connector: SnowflakeConnector = get_connector(connection_config) engine = connector.client() diff --git a/tests/ops/models/test_application_config.py b/tests/ops/models/test_application_config.py index 468478a507..524dfdc452 100644 --- a/tests/ops/models/test_application_config.py +++ b/tests/ops/models/test_application_config.py @@ -221,7 +221,7 @@ def test_update_config_set(self, db): ApplicationConfig.update_config_set(db, CONFIG) # check a few specific config properties of different types on the database record db_config = ApplicationConfig.get_config_set(db) - assert dumps(db_config) == CONFIG.json() + assert dumps(db_config, separators=(",", ":")) == CONFIG.model_dump_json() # change a few config values, set the db record, ensure it's updated notification_service_type = CONFIG.notifications.notification_service_type @@ -244,7 +244,7 @@ def test_update_config_set(self, db): # now we update the db record and all values should align again ApplicationConfig.update_config_set(db, CONFIG) db_config = ApplicationConfig.get_config_set(db) - assert dumps(db_config) == CONFIG.json() + assert dumps(db_config, separators=(",", ":")) == CONFIG.model_dump_json() # reset config values to initial values to ensure we don't mess up any state CONFIG.notifications.notification_service_type = notification_service_type diff --git a/tests/ops/models/test_connectionconfig.py b/tests/ops/models/test_connectionconfig.py index b3a2bc3ae3..93978ab8fc 100644 --- a/tests/ops/models/test_connectionconfig.py +++ b/tests/ops/models/test_connectionconfig.py @@ -124,7 +124,7 @@ def test_default_value_saas_config( "access": AccessLevel.read, }, ) - saas_config = SaaSConfig(**saas_example_config) + saas_config = SaaSConfig.model_validate(saas_example_config) assert connection_config.secrets is None # verify that setting the SaaS config for the first time populates diff --git a/tests/ops/models/test_consent_request.py b/tests/ops/models/test_consent_request.py index b13ab97cd9..253f3f5617 100644 --- a/tests/ops/models/test_consent_request.py +++ b/tests/ops/models/test_consent_request.py @@ -135,7 +135,7 @@ def test_queue_privacy_request_to_propagate_consent( PrivacyRequestResponse( id="fake_privacy_request_id", status=PrivacyRequestStatus.pending, - policy=PolicyResponse.from_orm(consent_policy), + policy=PolicyResponse.model_validate(consent_policy), ) ], failed=[], @@ -179,6 +179,13 @@ def test_queue_privacy_request_to_propagate_consent( call_kwargs["authenticated"] is True ), "We already validated identity with a verification code earlier in the request" + for label, value in call_kwargs["data"][ + 0 + ].custom_privacy_request_fields.items(): + call_kwargs["data"][0].custom_privacy_request_fields[label] = ( + value.model_dump(mode="json") + ) + assert call_kwargs["data"][0].custom_privacy_request_fields == custom_fields provided_identity.delete(db) @@ -200,7 +207,7 @@ def test_do_not_queue_privacy_request_if_no_executable_preferences( PrivacyRequestResponse( id="fake_privacy_request_id", status=PrivacyRequestStatus.pending, - policy=PolicyResponse.from_orm(consent_policy), + policy=PolicyResponse.model_validate(consent_policy), ) ], failed=[], @@ -248,7 +255,7 @@ def test_merge_in_browser_identity_with_provided_identity( PrivacyRequestResponse( id="fake_privacy_request_id", status=PrivacyRequestStatus.pending, - policy=PolicyResponse.from_orm(consent_policy), + policy=PolicyResponse.model_validate(consent_policy), ) ], failed=[], @@ -285,6 +292,11 @@ def test_merge_in_browser_identity_with_provided_identity( assert identity_of_privacy_request.email == "test@email.com" assert identity_of_privacy_request.ga_client_id == browser_identity.ga_client_id - assert call_kwargs["data"][0].custom_privacy_request_fields == custom_fields + for label, value in call_kwargs["data"][ + 0 + ].custom_privacy_request_fields.items(): + call_kwargs["data"][0].custom_privacy_request_fields[label] = ( + value.model_dump(mode="json") + ) provided_identity.delete(db) diff --git a/tests/ops/models/test_datasetconfig.py b/tests/ops/models/test_datasetconfig.py index 6949e748c6..5c52b7eca0 100644 --- a/tests/ops/models/test_datasetconfig.py +++ b/tests/ops/models/test_datasetconfig.py @@ -283,7 +283,9 @@ def test_no_existing_dataset_config_or_ctl_dataset( # we need to do the same instantiation here, i.e. on the test side of the fence # to make our assertions more straightforward postgres_dataset_result = Dataset(**postgres_dataset) - assert ctl_dataset.collections[0] == postgres_dataset_result.collections[0] + assert ctl_dataset.collections[0] == postgres_dataset_result.collections[ + 0 + ].model_dump(mode="json") dataset_config.delete(db) ctl_dataset.delete(db) @@ -358,7 +360,9 @@ def test_no_existing_dataset_config_but_ctl_dataset_exists( # we need to do the same instantiation here, i.e. on the test side of the fence # to make our assertions more straightforward dataset_result = Dataset(**dataset_data) - assert ctl_dataset.collections[0] == dataset_result.collections[0] + assert ctl_dataset.collections[0] == dataset_result.collections[0].model_dump( + mode="json" + ) dataset_config.delete(db) ctl_dataset.delete(db) @@ -423,4 +427,6 @@ def test_existing_dataset_config_and_ctl_dataset(self, dataset_config, db): # we need to do the same instantiation here, i.e. on the test side of the fence # to make our assertions more straightforward dataset_result = Dataset(**dataset_data) - assert updated_ctl_dataset.collections[0] == dataset_result.collections[0] + assert updated_ctl_dataset.collections[0] == dataset_result.collections[ + 0 + ].model_dump(mode="json") diff --git a/tests/ops/models/test_privacy_notice.py b/tests/ops/models/test_privacy_notice.py index 9b6500448f..933ef93f38 100644 --- a/tests/ops/models/test_privacy_notice.py +++ b/tests/ops/models/test_privacy_notice.py @@ -608,7 +608,7 @@ def test_relevant_cookies( ) assert [ - CookieSchema.from_orm(cookie) for cookie in privacy_notice.cookies + CookieSchema.model_validate(cookie) for cookie in privacy_notice.cookies ] == expected_cookies, description def test_generate_privacy_notice_key(self, privacy_notice): diff --git a/tests/ops/models/test_privacy_request.py b/tests/ops/models/test_privacy_request.py index 605a1f9fbc..6553bc3b6d 100644 --- a/tests/ops/models/test_privacy_request.py +++ b/tests/ops/models/test_privacy_request.py @@ -292,7 +292,7 @@ def test_trigger_one_way_policy_webhook( "privacy_request_id": privacy_request.id, "direction": webhook.direction.value, "callback_type": webhook.prefix, - "identity": identity.dict(), + "identity": identity.model_dump(mode="json"), }, status_code=500, ) @@ -319,7 +319,7 @@ def test_trigger_two_way_policy_webhook_with_error( "privacy_request_id": privacy_request.id, "direction": webhook.direction.value, "callback_type": webhook.prefix, - "identity": identity.dict(), + "identity": identity.model_dump(mode="json"), }, status_code=500, ) @@ -345,7 +345,7 @@ def test_trigger_two_way_policy_webhook_200_proceed( "privacy_request_id": privacy_request.id, "direction": webhook.direction.value, "callback_type": webhook.prefix, - "identity": identity.dict(), + "identity": identity.model_dump(mode="json"), "halt": False, }, status_code=200, @@ -373,7 +373,7 @@ def test_trigger_two_way_policy_webhook_200_pause( "privacy_request_id": privacy_request.id, "direction": webhook.direction.value, "callback_type": webhook.prefix, - "identity": identity.dict(), + "identity": identity.model_dump(mode="json"), "halt": True, }, status_code=200, @@ -401,7 +401,7 @@ def test_trigger_two_way_policy_webhook_add_derived_identity( "privacy_request_id": privacy_request.id, "direction": webhook.direction.value, "callback_type": webhook.prefix, - "identity": identity.dict(), + "identity": identity.model_dump(mode="json"), "derived_identity": {"phone_number": "+5555555555"}, "halt": False, }, @@ -435,7 +435,7 @@ def test_two_way_validation_issues( "privacy_request_id": privacy_request.id, "direction": webhook.direction.value, "callback_type": webhook.prefix, - "identity": identity.dict(), + "identity": identity.model_dump(mode="json"), }, status_code=200, ) @@ -450,7 +450,7 @@ def test_two_way_validation_issues( "privacy_request_id": privacy_request.id, "direction": webhook.direction.value, "callback_type": webhook.prefix, - "identity": identity.dict(), + "identity": identity.model_dump(mode="json"), "derived_identity": {"unsupported_identity": "1200 Fides Road"}, "halt": True, }, diff --git a/tests/ops/models/test_property.py b/tests/ops/models/test_property.py index 170f071b4a..d79ef71b19 100644 --- a/tests/ops/models/test_property.py +++ b/tests/ops/models/test_property.py @@ -41,12 +41,12 @@ def test_create_property(self, db, minimal_experience, privacy_center_config): privacy_center_config=privacy_center_config, stylesheet=":root:root { --chakra-colors-gray-50: #fff9ea; }", paths=["test"], - ).dict(), + ).model_dump(), ) assert prop.name == "New Property" assert prop.type == PropertyType.website assert prop.id.startswith("FDS") - assert prop.privacy_center_config == privacy_center_config + assert prop.privacy_center_config == privacy_center_config.model_dump() assert prop.stylesheet == ":root:root { --chakra-colors-gray-50: #fff9ea; }" assert prop.paths == ["test"] assert prop.is_default is True @@ -69,12 +69,12 @@ def test_create_second_property(self, db, privacy_center_config, property_a): privacy_center_config=privacy_center_config, stylesheet=":root:root { --chakra-colors-gray-50: #fff9ea; }", paths=["testing"], - ).dict(), + ).model_dump(), ) assert prop.name == "New Property 2" assert prop.type == PropertyType.website assert prop.id.startswith("FDS") - assert prop.privacy_center_config == privacy_center_config + assert prop.privacy_center_config == privacy_center_config.model_dump() assert prop.stylesheet == ":root:root { --chakra-colors-gray-50: #fff9ea; }" assert prop.paths == ["testing"] assert prop.is_default is False @@ -95,7 +95,7 @@ def test_create_property_duplicate_paths( privacy_center_config=privacy_center_config, stylesheet=":root:root { --chakra-colors-gray-50: #fff9ea; }", paths=["test"], - ).dict(), + ).model_dump(), ) with pytest.raises(ValueError): @@ -109,7 +109,7 @@ def test_create_property_duplicate_paths( privacy_center_config=privacy_center_config, stylesheet=":root:root { --chakra-colors-gray-50: #fff9ea; }", paths=["test"], - ).dict(), + ).model_dump(), ) second_prop = Property.filter( @@ -128,7 +128,7 @@ def test_create_property_with_special_characters(self, db): experiences=[], messaging_templates=[], paths=[], - ).dict(), + ).model_dump(), ) assert prop.name == "New Property (Prod)" assert prop.type == PropertyType.website @@ -183,7 +183,7 @@ def test_update_property_duplicate_paths(self, db: Session): experiences=[], messaging_templates=[], paths=["test"], - ).dict(), + ).model_dump(), ) second_prop = Property.create( @@ -194,7 +194,7 @@ def test_update_property_duplicate_paths(self, db: Session): experiences=[], messaging_templates=[], paths=[], - ).dict(), + ).model_dump(), ) with pytest.raises(ValueError): @@ -216,7 +216,7 @@ def test_property_paths_are_deleted_on_property_update( privacy_center_config=privacy_center_config, stylesheet=":root:root { --chakra-colors-gray-50: #fff9ea; }", paths=["first", "second", "third"], - ).dict(), + ).model_dump(), ) property_paths = PropertyPath.filter( diff --git a/tests/ops/models/test_saas_config.py b/tests/ops/models/test_saas_config.py index e336654efe..c66c7e695d 100644 --- a/tests/ops/models/test_saas_config.py +++ b/tests/ops/models/test_saas_config.py @@ -245,7 +245,7 @@ def test_list_default_values(self): def test_missing_name(self): with pytest.raises(ValidationError) as exc: ConnectorParam(default_value="checking") - assert "field required" in str(exc.value) + assert "Field required" in str(exc.value) def test_default_value_not_in_options(self): with pytest.raises(ValidationError) as exc: diff --git a/tests/ops/models/test_storage.py b/tests/ops/models/test_storage.py index cb0391b9f9..a749bc1a2b 100644 --- a/tests/ops/models/test_storage.py +++ b/tests/ops/models/test_storage.py @@ -120,7 +120,9 @@ def test_unique_key_constraint( storage_incoming_one.name = storage_config.name with pytest.raises(KeyOrNameAlreadyExists): - StorageConfig.create_or_update(db=db, data=storage_incoming_one.dict()) + StorageConfig.create_or_update( + db=db, data=storage_incoming_one.model_dump(mode="json") + ) def test_create_storage_config( self, @@ -129,7 +131,7 @@ def test_create_storage_config( storage_details_s3, ): storage_config = StorageConfig.create_or_update( - db=db, data=storage_incoming_one.dict() + db=db, data=storage_incoming_one.model_dump(mode="json") ) assert storage_config.name == "test storage destination 1" @@ -295,6 +297,6 @@ def test_format_validator(self): details={StorageDetails.NAMING.value: FileNaming.request_id.value}, ) assert ( - "Only JSON or HTML upload format are supported for local storage destinations." - in str(e) + "Value error, Only JSON or HTML upload format are supported for local storage destinations." + in str(e._excinfo[1]) ) diff --git a/tests/ops/schemas/connection_configuration/test_connection_config.py b/tests/ops/schemas/connection_configuration/test_connection_config.py index 078849be74..d3a9fc3362 100644 --- a/tests/ops/schemas/connection_configuration/test_connection_config.py +++ b/tests/ops/schemas/connection_configuration/test_connection_config.py @@ -5,7 +5,7 @@ ) -class TestMaskSenstiveValues: +class TestMaskSensitiveValues: @pytest.fixture(scope="function") def secret_schema(self): return { diff --git a/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py b/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py index 077f311e20..176b8148cf 100644 --- a/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py +++ b/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py @@ -34,13 +34,14 @@ def test_validation( ): schema = SaaSSchemaFactory(saas_config).get_saas_schema() config = saas_example_secrets - schema.parse_obj(config) + schema.model_validate(config) def test_missing_fields(self, saas_config: SaaSConfig): schema = SaaSSchemaFactory(saas_config).get_saas_schema() config = {"domain": "domain", "username": "username"} with pytest.raises(ValidationError) as exc: - schema.parse_obj(config) + schema.model_validate(config) + required_fields = [ connector_param.name for connector_param in ( @@ -51,9 +52,11 @@ def test_missing_fields(self, saas_config: SaaSConfig): ) # external refs are required or not connector_param.default_value ] + + errors = exc._excinfo[1].errors() assert ( - f"{saas_config.type}_schema must be supplied all of: " - f"[{', '.join(required_fields)}]." in str(exc.value) + errors[0]["msg"] + == "Value error, custom_schema must be supplied all of: [username, api_key, api_version, page_size, account_types, customer_id]." ) def test_extra_fields( @@ -65,7 +68,7 @@ def test_extra_fields( **saas_example_secrets, "extra": "extra", } - schema.parse_obj(config) + schema.model_validate(config) def test_default_value_fields( self, saas_config: SaaSConfig, saas_example_secrets: Dict[str, Any] @@ -79,7 +82,7 @@ def test_default_value_fields( assert domain_param.default_value del saas_example_secrets["domain"] config = saas_example_secrets - schema.parse_obj(config) + schema.model_validate(config) def test_value_in_options(self, saas_config: SaaSConfig): saas_config.connector_params = [ @@ -87,7 +90,7 @@ def test_value_in_options(self, saas_config: SaaSConfig): ] saas_config.external_references = [] schema = SaaSSchemaFactory(saas_config).get_saas_schema() - schema.parse_obj({"account_type": "checking"}) + schema.model_validate({"account_type": "checking"}) def test_value_in_options_with_multiselect(self, saas_config: SaaSConfig): saas_config.connector_params = [ @@ -97,7 +100,7 @@ def test_value_in_options_with_multiselect(self, saas_config: SaaSConfig): ] saas_config.external_references = [] schema = SaaSSchemaFactory(saas_config).get_saas_schema() - schema.parse_obj({"account_type": ["checking", "savings"]}) + schema.model_validate({"account_type": ["checking", "savings"]}) def test_value_not_in_options(self, saas_config: SaaSConfig): saas_config.connector_params = [ @@ -106,7 +109,7 @@ def test_value_not_in_options(self, saas_config: SaaSConfig): saas_config.external_references = [] schema = SaaSSchemaFactory(saas_config).get_saas_schema() with pytest.raises(ValidationError) as exc: - schema.parse_obj({"account_type": "investment"}) + schema.model_validate({"account_type": "investment"}) assert "'account_type' must be one of [checking, savings]" in str(exc.value) def test_value_not_in_options_with_multiselect(self, saas_config: SaaSConfig): @@ -118,7 +121,7 @@ def test_value_not_in_options_with_multiselect(self, saas_config: SaaSConfig): saas_config.external_references = [] schema = SaaSSchemaFactory(saas_config).get_saas_schema() with pytest.raises(ValidationError) as exc: - schema.parse_obj({"account_type": ["checking", "investment"]}) + schema.model_validate({"account_type": ["checking", "investment"]}) assert ( "[investment] are not valid options, 'account_type' must be a list of values from [checking, savings]" in str(exc.value) diff --git a/tests/ops/schemas/connection_configuration/test_email_schemas.py b/tests/ops/schemas/connection_configuration/test_email_schemas.py index aa3b847ddf..631536360c 100644 --- a/tests/ops/schemas/connection_configuration/test_email_schemas.py +++ b/tests/ops/schemas/connection_configuration/test_email_schemas.py @@ -43,7 +43,7 @@ def test_email_schema_invalid_email(self) -> None: identity_types=IdentityTypes(email=True, phone_number=False) ), ) - assert exc.value.errors()[0]["msg"] == "value is not a valid email address" + assert "value is not a valid email address" in exc.value.errors()[0]["msg"] def test_no_identities_supplied(self): with pytest.raises(ValueError) as exc: @@ -54,7 +54,10 @@ def test_no_identities_supplied(self): identity_types=IdentityTypes(email=False, phone_number=False) ), ) - assert exc.value.errors()[0]["msg"] == "Must supply at least one identity_type." + assert ( + exc.value.errors()[0]["msg"] + == "Value error, Must supply at least one identity_type." + ) def test_missing_third_party_vendor_name(self): with pytest.raises(ValueError) as exc: @@ -64,7 +67,7 @@ def test_missing_third_party_vendor_name(self): identity_types=IdentityTypes(email=True, phone_number=False) ), ) - assert exc.value.errors()[0]["msg"] == "field required" + assert exc.value.errors()[0]["msg"] == "Field required" assert exc.value.errors()[0]["loc"][0] == "third_party_vendor_name" def test_missing_recipient(self): @@ -75,7 +78,7 @@ def test_missing_recipient(self): identity_types=IdentityTypes(email=True, phone_number=False) ), ) - assert exc.value.errors()[0]["msg"] == "field required" + assert exc.value.errors()[0]["msg"] == "Field required" assert exc.value.errors()[0]["loc"][0] == "recipient_email_address" def test_extra_field(self): @@ -88,7 +91,7 @@ def test_extra_field(self): ), extra_field="extra_value", ) - assert exc.value.errors()[0]["msg"] == "extra fields not permitted" + assert exc.value.errors()[0]["msg"] == "Extra inputs are not permitted" class TestExtendedEmailSchema: diff --git a/tests/ops/schemas/test_identity_schema.py b/tests/ops/schemas/test_identity_schema.py index 8ee4f933b4..3bc822466d 100644 --- a/tests/ops/schemas/test_identity_schema.py +++ b/tests/ops/schemas/test_identity_schema.py @@ -28,7 +28,7 @@ def test_none_custom_identity(self): Identity( customer_id={"label": "Customer ID", "value": None}, ) - assert "none is not an allowed value" in str(exc.value) + assert "3 validation errors for LabeledIdentity" in str(exc.value) def test_invalid_custom_identity(self): with pytest.raises(ValueError) as exc: @@ -105,7 +105,7 @@ def test_invalid_custom_identity(self): ) def test_identity_dict(self, identity_data, expected_dict): identity = Identity(**identity_data) - assert identity.dict() == expected_dict + assert identity.model_dump(mode="json") == expected_dict @pytest.mark.parametrize( "identity_data, expected_dict", diff --git a/tests/ops/schemas/test_privacy_center_config.py b/tests/ops/schemas/test_privacy_center_config.py index b36c59b488..977ab975df 100644 --- a/tests/ops/schemas/test_privacy_center_config.py +++ b/tests/ops/schemas/test_privacy_center_config.py @@ -71,31 +71,34 @@ def test_invalid_executable_consent( consent_option.executable = True with pytest.raises(ValueError) as exc: - ConsentConfigPage(**consent_page.dict(by_alias=True)) + ConsentConfigPage(**consent_page.model_dump(by_alias=True)) assert "Cannot have more than one consent option be executable." in str( exc.value ) def test_serialize_by_alias(self, privacy_center_config: PrivacyCenterConfig): - assert "includeConsent" in privacy_center_config.dict(by_alias=True).keys() + assert ( + "includeConsent" in privacy_center_config.model_dump(by_alias=True).keys() + ) assert ( "consentOptions" - in privacy_center_config.consent.page.dict(by_alias=True).keys() + in privacy_center_config.consent.page.model_dump(by_alias=True).keys() ) for field in ["cookieKeys", "fidesDataUseKey"]: assert ( field in privacy_center_config.consent.page.consent_options[0] - .dict(by_alias=True) + .model_dump(by_alias=True) .keys() ) for field in ["confirmButtonText", "cancelButtonText", "modalTitle"]: assert ( - field in privacy_center_config.consent.button.dict(by_alias=True).keys() + field + in privacy_center_config.consent.button.model_dump(by_alias=True).keys() ) assert ( "globalPrivacyControl" in privacy_center_config.consent.page.consent_options[0] - .default.dict(by_alias=True) + .default.model_dump(by_alias=True) .keys() ) diff --git a/tests/ops/service/authentication/test_authentication_strategy_api_key.py b/tests/ops/service/authentication/test_authentication_strategy_api_key.py index b637b899a9..59169600b8 100644 --- a/tests/ops/service/authentication/test_authentication_strategy_api_key.py +++ b/tests/ops/service/authentication/test_authentication_strategy_api_key.py @@ -168,37 +168,40 @@ def test_api_key_auth_bad_config(): AuthenticationStrategy.get_strategy( "api_key", {"headers": ""} ).add_authentication(req, ConnectionConfig(secrets={})) - assert "not a valid list" in str(exc.value) + assert ( + "At least one 'header', 'query_param', or 'body' object must be defined in an 'api_key' auth configuration" + in str(exc.value) + ) with pytest.raises(FidesopsValidationError) as exc: AuthenticationStrategy.get_strategy( "api_key", {"headers": "foo"} ).add_authentication(req, ConnectionConfig(secrets={})) - assert "not a valid list" in str(exc.value) + assert "Input should be a valid list" in str(exc.value) with pytest.raises(FidesopsValidationError) as exc: AuthenticationStrategy.get_strategy( "api_key", {"headers": ["foo"]} ).add_authentication(req, ConnectionConfig(secrets={})) - assert "not a valid dict" in str(exc.value) + assert "Input should be a valid dictionary or instance of Header" in str(exc.value) with pytest.raises(FidesopsValidationError) as exc: AuthenticationStrategy.get_strategy( "api_key", {"headers": {"name": "token"}} ).add_authentication(req, ConnectionConfig()) - assert "not a valid list" in str(exc.value) + assert "Input should be a valid list" in str(exc.value) with pytest.raises(FidesopsValidationError) as exc: AuthenticationStrategy.get_strategy( "api_key", {"headers": [{"name": "token"}]} ).add_authentication(req, ConnectionConfig()) - assert "field required" in str(exc.value) + assert "Field required" in str(exc.value) with pytest.raises(FidesopsValidationError) as exc: AuthenticationStrategy.get_strategy( "api_key", {"headers": [{"value": ""}]} ).add_authentication(req, ConnectionConfig(secrets={})) - assert "field required" in str(exc.value) + assert "Field required" in str(exc.value) with pytest.raises(FidesopsValidationError) as exc: AuthenticationStrategy.get_strategy( @@ -210,7 +213,7 @@ def test_api_key_auth_bad_config(): ] }, ).add_authentication(req, ConnectionConfig(secrets={})) - assert "field required" in str(exc.value) + assert "Field required" in str(exc.value) with pytest.raises(FidesopsValidationError) as exc: AuthenticationStrategy.get_strategy( @@ -220,7 +223,7 @@ def test_api_key_auth_bad_config(): "query_params": {"name": "token"}, }, ).add_authentication(req, ConnectionConfig(secrets={})) - assert "not a valid list" in str(exc.value) + assert "Input should be a valid list" in str(exc.value) def test_api_key_auth_bad_param_value(): diff --git a/tests/ops/service/connectors/test_connector_registry_service.py b/tests/ops/service/connectors/test_connector_registry_service.py index bf55406805..70187b3a2a 100644 --- a/tests/ops/service/connectors/test_connector_registry_service.py +++ b/tests/ops/service/connectors/test_connector_registry_service.py @@ -357,7 +357,7 @@ def validate_updated_instances_additions( ) assert ( - DatasetCollection(**NEW_COLLECTION) + DatasetCollection(**NEW_COLLECTION).model_dump(mode="json") in updated_dataset_config.ctl_dataset.collections ) diff --git a/tests/ops/service/connectors/test_consent_email_connector.py b/tests/ops/service/connectors/test_consent_email_connector.py index 9d273ca446..e248a27820 100644 --- a/tests/ops/service/connectors/test_consent_email_connector.py +++ b/tests/ops/service/connectors/test_consent_email_connector.py @@ -367,7 +367,9 @@ def test_needs_email_old_workflow( privacy_request_with_consent_policy, ): privacy_request_with_consent_policy.consent_preferences = [ - Consent(data_use="marketing.advertising", opt_in=False).dict() + Consent(data_use="marketing.advertising", opt_in=False).model_dump( + mode="json" + ) ] assert ( test_sovrn_consent_email_connector.needs_email( @@ -429,7 +431,9 @@ def test_needs_email_unsupported_identity_old_workflow( self, test_sovrn_consent_email_connector, privacy_request_with_consent_policy ): privacy_request_with_consent_policy.consent_preferences = [ - Consent(data_use="marketing.advertising", opt_in=False).dict() + Consent(data_use="marketing.advertising", opt_in=False).model_dump( + mode="json" + ) ] assert ( test_sovrn_consent_email_connector.needs_email( @@ -538,7 +542,9 @@ def test_test_connection_call( assert call_kwargs["subject_name"] == "Sovrn" assert call_kwargs["required_identities"] == ["ljt_readerID"] - preferences = [pref.dict() for pref in call_kwargs["user_consent_preferences"]] + preferences = [ + pref.model_dump() for pref in call_kwargs["user_consent_preferences"] + ] assert len(preferences) == 1 assert preferences[0]["identities"] == {"ljt_readerID": "test_ljt_reader_id"} assert ( diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py index 81ad557a8b..83ee4db11f 100644 --- a/tests/ops/service/messaging/message_dispatch_service_test.py +++ b/tests/ops/service/messaging/message_dispatch_service_test.py @@ -573,8 +573,8 @@ def test_email_dispatch_twilio_sms_test_message( "+19198675309", ) - def test_fidesops_email_parse_object(self): - FidesopsMessage.parse_obj( + def test_fidesops_email_model_validateect(self): + FidesopsMessage.model_validate( { "action_type": MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT, "body_params": { @@ -585,7 +585,7 @@ def test_fidesops_email_parse_object(self): } ) - FidesopsMessage.parse_obj( + FidesopsMessage.model_validate( { "action_type": MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, "body_params": { diff --git a/tests/ops/service/pagination/test_pagination_strategy_link.py b/tests/ops/service/pagination/test_pagination_strategy_link.py index f22305b94a..46e753dfcc 100644 --- a/tests/ops/service/pagination/test_pagination_strategy_link.py +++ b/tests/ops/service/pagination/test_pagination_strategy_link.py @@ -135,7 +135,7 @@ def test_link_in_body_empty_string(response_with_empty_string_link): def test_wrong_source(): with pytest.raises(ValueError) as exc: LinkPaginationConfiguration(source="somewhere", path="links.next") - assert "value is not a valid enumeration member" in str(exc.value) + assert "Input should be 'headers' or 'body'" in str(exc.value) def test_config_mismatch(): diff --git a/tests/ops/service/privacy_request/test_email_batch_send.py b/tests/ops/service/privacy_request/test_email_batch_send.py index 4ed3fe4b8e..04510ad9bb 100644 --- a/tests/ops/service/privacy_request/test_email_batch_send.py +++ b/tests/ops/service/privacy_request/test_email_batch_send.py @@ -33,7 +33,7 @@ def cache_identity_and_consent_preferences(privacy_request, db, reader_id): identity = Identity(email="customer_1#@example.com", ljt_readerID=reader_id) privacy_request.cache_identity(identity) privacy_request.consent_preferences = [ - Consent(data_use="marketing.advertising", opt_in=False).dict() + Consent(data_use="marketing.advertising", opt_in=False).model_dump(mode="json") ] privacy_request.save(db) @@ -186,7 +186,9 @@ def test_send_consent_email_failure_old_workflow( identity = Identity(email="customer_1#@example.com", ljt_readerID="12345") privacy_request_awaiting_consent_email_send.cache_identity(identity) privacy_request_awaiting_consent_email_send.consent_preferences = [ - Consent(data_use="marketing.advertising", opt_in=False).dict() + Consent(data_use="marketing.advertising", opt_in=False).model_dump( + mode="json" + ) ] privacy_request_awaiting_consent_email_send.save(db) diff --git a/tests/ops/service/privacy_request/test_request_runner_service.py b/tests/ops/service/privacy_request/test_request_runner_service.py index ca07bbcb55..591a65322b 100644 --- a/tests/ops/service/privacy_request/test_request_runner_service.py +++ b/tests/ops/service/privacy_request/test_request_runner_service.py @@ -2292,8 +2292,8 @@ def test_run_webhooks_validation_error( privacy_request, policy_pre_execution_webhooks, ): - mock_trigger_policy_webhook.side_effect = ValidationError( - errors={}, model=SecondPartyResponseFormat + mock_trigger_policy_webhook.side_effect = ValidationError.from_exception_data( + title="Validation Error", line_errors=[] ) proceed = run_webhooks_and_report_status(db, privacy_request, PolicyPreWebhook) @@ -2874,7 +2874,9 @@ def test_privacy_request_is_put_in_awaiting_email_send_status_old_workflow( identity = Identity(email="customer_1#@example.com", ljt_readerID="12345") privacy_request_with_consent_policy.cache_identity(identity) privacy_request_with_consent_policy.consent_preferences = [ - Consent(data_use="marketing.advertising", opt_in=False).dict() + Consent(data_use="marketing.advertising", opt_in=False).model_dump( + mode="json" + ) ] privacy_request_with_consent_policy.save(db) @@ -2935,7 +2937,9 @@ def test_needs_batch_email_send_no_email_consent_connections_old_workflow( self, db, privacy_request_with_consent_policy ): privacy_request_with_consent_policy.consent_preferences = [ - Consent(data_use="marketing.advertising", opt_in=False).dict() + Consent(data_use="marketing.advertising", opt_in=False).model_dump( + mode="json" + ) ] privacy_request_with_consent_policy.save(db) assert not needs_batch_email_send( @@ -2958,7 +2962,9 @@ def test_needs_batch_email_send_no_relevant_identities_old_workflow( self, db, privacy_request_with_consent_policy ): privacy_request_with_consent_policy.consent_preferences = [ - Consent(data_use="marketing.advertising", opt_in=False).dict() + Consent(data_use="marketing.advertising", opt_in=False).model_dump( + mode="json" + ) ] privacy_request_with_consent_policy.save(db) assert not needs_batch_email_send( @@ -2982,7 +2988,9 @@ def test_needs_batch_email_send_old_workflow( self, db, privacy_request_with_consent_policy ): privacy_request_with_consent_policy.consent_preferences = [ - Consent(data_use="marketing.advertising", opt_in=False).dict() + Consent(data_use="marketing.advertising", opt_in=False).model_dump( + mode="json" + ) ] privacy_request_with_consent_policy.save(db) assert needs_batch_email_send( diff --git a/tests/ops/service/privacy_request/test_request_service.py b/tests/ops/service/privacy_request/test_request_service.py index 0a7dee78fa..6c94016420 100644 --- a/tests/ops/service/privacy_request/test_request_service.py +++ b/tests/ops/service/privacy_request/test_request_service.py @@ -1,7 +1,5 @@ -import json import time from datetime import datetime -from unittest import mock import pytest from httpx import HTTPStatusError diff --git a/tests/ops/service/test_strategy_retrieval.py b/tests/ops/service/test_strategy_retrieval.py index e515a8a7d0..e77ebd03d9 100644 --- a/tests/ops/service/test_strategy_retrieval.py +++ b/tests/ops/service/test_strategy_retrieval.py @@ -100,7 +100,7 @@ def test_valid_strategy(self): config = SomeStrategyConfiguration(some_key="non default value") retrieved_strategy = PostProcessorStrategy.get_strategy( - SomeStrategy.name, config.dict() + SomeStrategy.name, config.model_dump(mode="json") ) assert isinstance(retrieved_strategy, SomeStrategy) assert retrieved_strategy.some_config == "non default value" @@ -113,31 +113,31 @@ def test_multi_level_inheritance_strategy(self): config = SomeStrategyConfiguration(some_key="non default value") retrieved_strategy = PostProcessorStrategy.get_strategy( - SomeStrategy.name, config.dict() + SomeStrategy.name, config.model_dump(mode="json") ) assert isinstance(retrieved_strategy, SomeStrategy) retrieved_strategy = PostProcessorStrategy.get_strategy( - SomeSubStrategy.name, config.dict() + SomeSubStrategy.name, config.model_dump(mode="json") ) assert isinstance(retrieved_strategy, SomeSubStrategy) assert issubclass(type(retrieved_strategy), SomeStrategy) retrieved_strategy = PostProcessorStrategy.get_strategy( - AnotherSubStrategy.name, config.dict() + AnotherSubStrategy.name, config.model_dump(mode="json") ) assert isinstance(retrieved_strategy, AnotherSubStrategy) assert issubclass(type(retrieved_strategy), SomeStrategy) retrieved_strategy = PostProcessorStrategy.get_strategy( - SomeSubSubStrategy.name, config.dict() + SomeSubSubStrategy.name, config.model_dump(mode="json") ) assert isinstance(retrieved_strategy, SomeSubSubStrategy) assert issubclass(type(retrieved_strategy), SomeStrategy) assert issubclass(type(retrieved_strategy), SomeSubStrategy) retrieved_strategy = PostProcessorStrategy.get_strategy( - DifferentStrategySubClass.name, config.dict() + DifferentStrategySubClass.name, config.model_dump(mode="json") ) assert isinstance(retrieved_strategy, DifferentStrategySubClass) assert issubclass(type(retrieved_strategy), SomeAbstractStrategyClass) diff --git a/tests/ops/task/test_filter_results.py b/tests/ops/task/test_filter_results.py index fe951802c8..14432292fd 100644 --- a/tests/ops/task/test_filter_results.py +++ b/tests/ops/task/test_filter_results.py @@ -575,7 +575,11 @@ def test_filter_data_categories(self): } dataset_graph = DatasetGraph( - *[convert_dataset_to_graph(Dataset.parse_obj(dataset), "postgres_example")] + *[ + convert_dataset_to_graph( + Dataset.model_validate(dataset), "postgres_example" + ) + ] ) only_a_categories = filter_data_categories( @@ -670,7 +674,11 @@ def test_filter_data_categories_arrays(self): } dataset_graph = DatasetGraph( - *[convert_dataset_to_graph(Dataset.parse_obj(dataset), "postgres_example")] + *[ + convert_dataset_to_graph( + Dataset.model_validate(dataset), "postgres_example" + ) + ] ) only_a_category = filter_data_categories( @@ -1082,10 +1090,10 @@ def test_filter_data_categories_limited_results(self): dataset_graph = DatasetGraph( *[ convert_dataset_to_graph( - Dataset.parse_obj(postgres_dataset), "postgres_example" + Dataset.model_validate(postgres_dataset), "postgres_example" ), convert_dataset_to_graph( - Dataset.parse_obj(mongo_dataset), "mongo_test" + Dataset.model_validate(mongo_dataset), "mongo_test" ), ] ) @@ -1178,7 +1186,11 @@ def test_filter_by_collection_level_parent_data_category(self): } dataset_graph = DatasetGraph( - *[convert_dataset_to_graph(Dataset.parse_obj(dataset), "postgres_example")] + *[ + convert_dataset_to_graph( + Dataset.model_validate(dataset), "postgres_example" + ) + ] ) # here we filter by the `user` data category and since the collection-level @@ -1224,7 +1236,11 @@ def test_filter_by_collection_level_child_data_category(self): } dataset_graph = DatasetGraph( - *[convert_dataset_to_graph(Dataset.parse_obj(dataset), "postgres_example")] + *[ + convert_dataset_to_graph( + Dataset.model_validate(dataset), "postgres_example" + ) + ] ) # Here we filter by the `user.content` which is more specific than the collection-level @@ -1265,7 +1281,11 @@ def test_filter_by_collection_level_exact_data_category(self): } dataset_graph = DatasetGraph( - *[convert_dataset_to_graph(Dataset.parse_obj(dataset), "postgres_example")] + *[ + convert_dataset_to_graph( + Dataset.model_validate(dataset), "postgres_example" + ) + ] ) assert ( diff --git a/tests/ops/task/test_graph_task.py b/tests/ops/task/test_graph_task.py index fed7572917..f69c89eb10 100644 --- a/tests/ops/task/test_graph_task.py +++ b/tests/ops/task/test_graph_task.py @@ -728,7 +728,7 @@ def create_dataset(self, db, fides_key, connection_config): }, ], ) - ctl_dataset = CtlDataset(**ds.dict()) + ctl_dataset = CtlDataset(**ds.model_dump(mode="json")) db.add(ctl_dataset) db.commit() diff --git a/tests/ops/task/traversal_data.py b/tests/ops/task/traversal_data.py index 6201d5a4ee..20d3773e17 100644 --- a/tests/ops/task/traversal_data.py +++ b/tests/ops/task/traversal_data.py @@ -771,19 +771,19 @@ def scylladb_dataset_dict(db_name: str) -> Dict[str, Any]: def postgres_db_graph_dataset(db_name: str, connection_key) -> GraphDataset: dataset = postgres_dataset_dict(db_name) - return convert_dataset_to_graph(Dataset.parse_obj(dataset), connection_key) + return convert_dataset_to_graph(Dataset.model_validate(dataset), connection_key) def scylla_db_graph_dataset(db_name: str) -> GraphDataset: dataset = scylladb_dataset_dict(db_name) - return convert_dataset_to_graph(Dataset.parse_obj(dataset), db_name) + return convert_dataset_to_graph(Dataset.model_validate(dataset), db_name) def mongo_db_graph_dataset( mongo_db_name: str, postgres_db_name: str, connection_key: str ) -> GraphDataset: dataset = mongo_dataset_dict(mongo_db_name, postgres_db_name) - return convert_dataset_to_graph(Dataset.parse_obj(dataset), connection_key) + return convert_dataset_to_graph(Dataset.model_validate(dataset), connection_key) def integration_db_mongo_graph( diff --git a/tests/ops/test_helpers/dataset_utils.py b/tests/ops/test_helpers/dataset_utils.py index 08d03611fb..e60efb9892 100644 --- a/tests/ops/test_helpers/dataset_utils.py +++ b/tests/ops/test_helpers/dataset_utils.py @@ -31,7 +31,7 @@ def update_dataset( """ generated_dataset = generate_dataset( - Dataset.from_orm(dataset_config.ctl_dataset).dict(), + Dataset.model_validate(dataset_config.ctl_dataset).model_dump(mode="json"), api_data, [endpoint["name"] for endpoint in connection_config.saas_config["endpoints"]], ) diff --git a/tests/ops/util/test_connection_type.py b/tests/ops/util/test_connection_type.py index cd1d1776c8..da3a659560 100644 --- a/tests/ops/util/test_connection_type.py +++ b/tests/ops/util/test_connection_type.py @@ -10,7 +10,7 @@ def test_get_connection_types(): - data = get_connection_types() + data = [obj.model_dump(mode="json") for obj in get_connection_types()] assert ( len(data) == len(ConnectionType) + len(ConnectorRegistry.connector_types()) - 4 ) # there are 4 connection types that are not returned by the endpoint @@ -38,10 +38,10 @@ def test_get_connection_types(): ], } in data - assert "saas" not in [item.identifier for item in data] - assert "https" not in [item.identifier for item in data] - assert "custom" not in [item.identifier for item in data] - assert "manual" not in [item.identifier for item in data] + assert "saas" not in [item["identifier"] for item in data] + assert "https" not in [item["identifier"] for item in data] + assert "custom" not in [item["identifier"] for item in data] + assert "manual" not in [item["identifier"] for item in data] assert { "identifier": ConnectionType.sovrn.value, @@ -288,7 +288,10 @@ def connection_type_objects(): def test_get_connection_types_action_type_filter( action_types, assert_in_data, assert_not_in_data, connection_type_objects ): - data = get_connection_types(action_types=action_types) + data = [ + obj.model_dump(mode="json") + for obj in get_connection_types(action_types=action_types) + ] for connection_type in assert_in_data: obj = connection_type_objects[connection_type] diff --git a/tests/ops/util/test_dataset_yaml.py b/tests/ops/util/test_dataset_yaml.py index 5e8a7c1ccc..771a95e471 100644 --- a/tests/ops/util/test_dataset_yaml.py +++ b/tests/ops/util/test_dataset_yaml.py @@ -128,7 +128,7 @@ def __to_dataset__(yamlstr: str) -> Dict[str, Any]: def test_dataset_yaml_format(): """Test that 'after' parameters are properly read""" dataset = __to_dataset__(example_dataset_yaml) - d: Dataset = Dataset.parse_obj(dataset) + d: Dataset = Dataset.model_validate(dataset) config = convert_dataset_to_graph(d, "ignore") assert config.after == {"db1", "db2", "db3"} assert config.collections[0].after == { @@ -143,7 +143,7 @@ def test_dataset_yaml_format_invalid_format(): dataset = __to_dataset__(example_dataset_yaml) dataset.get("collections")[0].get("fidesops_meta").get("after")[0] = "invalid" with pytest.raises(ValueError) as exc: - d: Dataset = Dataset.parse_obj(dataset) + d: Dataset = Dataset.model_validate(dataset) convert_dataset_to_graph(d, "ignore") assert "FidesCollection must be specified in the form 'FidesKey.FidesKey'" in str( exc.value @@ -157,7 +157,7 @@ def test_dataset_yaml_format_invalid_fides_keys(): 0 ] = "invalid*dataset*name.invalid*collection*name" with pytest.raises(ValueError) as exc: - d: Dataset = Dataset.parse_obj(dataset) + d: Dataset = Dataset.model_validate(dataset) convert_dataset_to_graph(d, "ignore") assert ( "FidesKeys must only contain alphanumeric characters, '.', '_', '<', '>' or '-'." @@ -167,7 +167,7 @@ def test_dataset_yaml_format_invalid_fides_keys(): def test_nested_dataset_format(): dataset = __to_dataset__(example_dataset_nested_yaml) - ds = Dataset.parse_obj(dataset) + ds = Dataset.model_validate(dataset) graph = convert_dataset_to_graph(ds, "ignore") comments_field = field([graph], "mongo_nested_test", "photos", "comments") @@ -194,7 +194,7 @@ def test_nested_dataset_format(): def test_nested_dataset_validation(): with pytest.raises(ValidationError): - Dataset.parse_obj(__to_dataset__(example_bad_dataset_nested_yaml)) + Dataset.model_validate(__to_dataset__(example_bad_dataset_nested_yaml)) def test_invalid_datatype(): @@ -210,7 +210,7 @@ def test_invalid_datatype(): data_type: this_is_bad""" dataset = __to_dataset__(bad_data_declaration) with pytest.raises(ValidationError): - Dataset.parse_obj(dataset) + Dataset.model_validate(dataset) example_postgres_yaml = """dataset: @@ -255,11 +255,11 @@ def test_invalid_datatype(): def test_dataset_graph_connected_by_nested_fields(): """Two of the fields in the postgres dataset references a nested field in the mongo dataset""" dataset = __to_dataset__(example_dataset_nested_yaml) - ds = Dataset.parse_obj(dataset) + ds = Dataset.model_validate(dataset) mongo_dataset = convert_dataset_to_graph(ds, "ignore") postgres_dataset = __to_dataset__(example_postgres_yaml) - ds_postgres = Dataset.parse_obj(postgres_dataset) + ds_postgres = Dataset.model_validate(postgres_dataset) postgres_dataset = convert_dataset_to_graph(ds_postgres, "ignore") dataset_graph = DatasetGraph(mongo_dataset, postgres_dataset) @@ -314,7 +314,7 @@ def test_dataset_graph_connected_by_nested_fields(): def test_object_data_category_validation(): """Test trying to validate object with data category specified""" with pytest.raises(ValidationError): - Dataset.parse_obj( + Dataset.model_validate( __to_dataset__(example_object_with_data_categories_nested_yaml) ) @@ -341,7 +341,7 @@ def test_object_data_category_validation(): def test_return_all_elements_specified_on_non_array_field(): """Test return_all_elements can only be specified on array fields""" with pytest.raises(ValidationError): - Dataset.parse_obj(__to_dataset__(non_array_field_with_invalid_flag)) + Dataset.model_validate(__to_dataset__(non_array_field_with_invalid_flag)) skip_processing_yaml = """dataset: @@ -397,7 +397,7 @@ def test_collection_skip_processing(self): """A skip_processing flag at collection > fides_meta causes the collection to be ignored in convert_dataset_to_graph""" dataset = __to_dataset__(skip_processing_yaml) - ds = Dataset.parse_obj(dataset) + ds = Dataset.model_validate(dataset) converted_dataset = convert_dataset_to_graph(ds, "ignore") assert len(converted_dataset.collections) == 1 assert converted_dataset.collections[0].name == "b_collection" @@ -409,7 +409,7 @@ def test_invalid_collection_skip_processing(self): """Skipping a collection that shouldn't be skipped is not picked up in convert_dataset_to_graph, but downstream when DatasetGraph is instantiated""" dataset = __to_dataset__(skip_processing_invalid_yaml) - ds = Dataset.parse_obj(dataset) + ds = Dataset.model_validate(dataset) converted_dataset = convert_dataset_to_graph(ds, "ignore") assert len(converted_dataset.collections) == 1 assert converted_dataset.collections[0].name == "b_collection" diff --git a/tests/ops/util/test_storage_util.py b/tests/ops/util/test_storage_util.py index 8d263d3a51..ac43f1535f 100644 --- a/tests/ops/util/test_storage_util.py +++ b/tests/ops/util/test_storage_util.py @@ -33,7 +33,7 @@ def test_get_schema_for_secrets_invalid_storage_type(self): "fake_key": "aws_secret_access_key", }, ) - assert "extra fields not permitted" in str(e) + assert "Extra inputs are not permitted" in str(e) def test_get_schema_for_secrets(self): secrets = get_schema_for_secrets(