diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 29064c9542..3128c03794 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -17,6 +17,8 @@ _Write some things here about the changes and any potential caveats_ * [ ] Issue requirements met * [ ] All CI pipelines succeeded * [ ] `CHANGELOG.md` updated + * [ ] Add a https://github.com/ethyca/fides/labels/db-migration label to the entry if your change includes a DB migration + * [ ] Add a https://github.com/ethyca/fides/labels/high-risk label to the entry if your change includes a high-risk change (i.e. potential for performance impact or unexpected regression) that should be flagged * Followup issues: * [ ] Followup issues created (include link) * [ ] No followup issues diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml index e1cc215070..065a8ef920 100644 --- a/.github/workflows/backend_checks.yml +++ b/.github/workflows/backend_checks.yml @@ -75,43 +75,6 @@ jobs: path: /tmp/python-${{ matrix.python_version }}.tar retention-days: 1 - ################### - ## Static Checks ## - ################### - Static-Checks: - strategy: - matrix: - session_name: - [ - '"isort(check)"', - '"black(check)"', - "mypy", - "pylint", - "xenon", - "check_install", - '"pytest(nox)"', - ] - runs-on: ubuntu-latest - continue-on-error: true - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set Up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - cache: "pip" - - - name: Install Nox - run: pip install nox>=2022 - - - name: Install Dev Requirements - run: pip install -r dev-requirements.txt - - - name: Run Static Check - run: nox -s ${{ matrix.session_name }} - ################## ## Performance ## ################## diff --git a/.github/workflows/static_checks.yml b/.github/workflows/static_checks.yml new file mode 100644 index 0000000000..8a26cfb6ff --- /dev/null +++ b/.github/workflows/static_checks.yml @@ -0,0 +1,53 @@ +name: Backend Static Code Checks + +on: + pull_request: + push: + branches: + - "main" + - "release-**" + +env: + IMAGE: ethyca/fides:local + DEFAULT_PYTHON_VERSION: "3.10.13" + # Docker auth with read-only permissions. + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_RO_TOKEN: ${{ secrets.DOCKER_RO_TOKEN }} + +jobs: + ################### + ## Static Checks ## + ################### + Static-Checks: + strategy: + matrix: + session_name: + [ + '"isort(check)"', + '"black(check)"', + "mypy", + "pylint", + "xenon", + "check_install", + '"pytest(nox)"', + ] + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set Up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: "pip" + + - name: Install Nox + run: pip install nox>=2022 + + - name: Install Dev Requirements + run: pip install -r dev-requirements.txt + + - name: Run Static Check + run: nox -s ${{ matrix.session_name }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 325bba29cd..1862215a98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,9 +15,20 @@ The types of changes are: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +Changes can also be flagged with a GitHub label for tracking purposes. The URL of the label should be put at the end of the entry. The possible labels are: +- https://github.com/ethyca/fides/labels/high-risk: to indicate that a change is a "high-risk" change that could potentially lead to unanticipated regressions or degradations +- https://github.com/ethyca/fides/labels/db-migration: to indicate that a given change includes a DB migration + ## [Unreleased](https://github.com/ethyca/fides/compare/2.52.0...main) +### Added +- Added Action Center MVP behind new feature flag [#5622](https://github.com/ethyca/fides/pull/5622) +- Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629) +### Fixed +- Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) +- Fixed column ordering issue in the Data Map report [#5649](https://github.com/ethyca/fides/pull/5649) +- Fixed issue where the Data Map report filter dialog was missing an Accordion item label [#5649](https://github.com/ethyca/fides/pull/5649) ## [2.52.0](https://github.com/ethyca/fides/compare/2.51.2...2.52.0) @@ -26,7 +37,7 @@ The types of changes are: - Added event based communication example to the Cookie House sample app [#5597](https://github.com/ethyca/fides/pull/5597) - Added new erasure tests for BigQuery Enterprise [#5554](https://github.com/ethyca/fides/pull/5554) - Added new `has_next` parameter for the `link` pagination strategy [#5596](https://github.com/ethyca/fides/pull/5596) -- Added a `DBCache` model for database-backed caching [#5613](https://github.com/ethyca/fides/pull/5613) +- Added a `DBCache` model for database-backed caching [#5613](https://github.com/ethyca/fides/pull/5613) https://github.com/ethyca/fides/labels/db-migration - Adds "reclassify" button to discovery result tables [#5574](https://github.com/ethyca/fides/pull/5574) - Added support for exporting datamaps with column renaming, reordering and visibility options [#5543](https://github.com/ethyca/fides/pull/5543) @@ -95,7 +106,7 @@ The types of changes are: - Allow hiding systems via a `hidden` parameter and add two flags on the `/system` api endpoint; `show_hidden` and `dnd_relevant`, to display only systems with integrations [#5484](https://github.com/ethyca/fides/pull/5484) - The CMP override `fides_privacy_policy_url` will now apply even if the `fides_override_language` doesn't match [#5515](https://github.com/ethyca/fides/pull/5515) - Updated POST taxonomy endpoints to handle creating resources without specifying fides_key [#5468](https://github.com/ethyca/fides/pull/5468) -- Disabled connection pooling for task workers and added retries and keep-alive configurations for database connections [#5448](https://github.com/ethyca/fides/pull/5448) +- Disabled connection pooling for task workers and added retries and keep-alive configurations for database connections [#5448](https://github.com/ethyca/fides/pull/5448) https://github.com/ethyca/fides/labels/high-risk - Added timeout handling in the UI for async discovery monitor-related queries [#5519](https://github.com/ethyca/fides/pull/5519) ### Developer Experience diff --git a/clients/admin-ui/cypress/e2e/action-center.cy.ts b/clients/admin-ui/cypress/e2e/action-center.cy.ts new file mode 100644 index 0000000000..1f816ae3b3 --- /dev/null +++ b/clients/admin-ui/cypress/e2e/action-center.cy.ts @@ -0,0 +1,175 @@ +import { stubActionCenter, stubPlus } from "cypress/support/stubs"; + +import { + ACTION_CENTER_ROUTE, + INTEGRATION_MANAGEMENT_ROUTE, +} from "~/features/common/nav/v2/routes"; + +describe("Action center", () => { + beforeEach(() => { + cy.login(); + stubPlus(true); + stubActionCenter(); + }); + + describe("disabled web monitor", () => { + beforeEach(() => { + cy.intercept("GET", "/api/v1/config*", { + body: { + detection_discovery: { + website_monitor_enabled: false, + }, + }, + }).as("getTranslationConfig"); + cy.visit(ACTION_CENTER_ROUTE); + }); + it("should display a message that the web monitor is disabled", () => { + cy.wait("@getTranslationConfig"); + cy.contains("currently disabled").should("exist"); + }); + }); + + describe("empty action center", () => { + beforeEach(() => { + cy.intercept("GET", "/api/v1/plus/discovery-monitor/aggregate-results*", { + fixture: "empty-pagination.json", + }).as("getMonitorResults"); + cy.visit(ACTION_CENTER_ROUTE); + }); + it("should display empty state", () => { + cy.wait("@getMonitorResults"); + cy.get("[data-testid='search-bar']").should("exist"); + cy.get(`[class*='ant-empty'] [class*='ant-empty-image']`).should("exist"); + cy.get( + `[class*='ant-empty'] a[href="${INTEGRATION_MANAGEMENT_ROUTE}"]`, + ).should("exist"); + }); + }); + + describe("Action center monitor aggregate results", () => { + const webMonitorKey = "my_web_monitor_2"; + const integrationMonitorKey = "My_New_BQ_Monitor"; + beforeEach(() => { + cy.visit(ACTION_CENTER_ROUTE); + }); + it("should render the current monitor results", () => { + cy.get("[data-testid='Action center']").should("exist"); + cy.wait("@getMonitorResults"); + cy.get("[data-testid*='monitor-result-']").should("have.length", 3); + cy.get("[data-testid^='monitor-result-']").each((result) => { + const monitorKey = result + .attr("data-testid") + .replace("monitor-result-", ""); + // linked title + cy.wrap(result) + .contains("assets detected") + .should("have.attr", "href", `${ACTION_CENTER_ROUTE}/${monitorKey}`); + // last monitored relative date with real date in tooltip + cy.wrap(result) + .find("[data-testid='monitor-date']") + .contains(" ago") + .realHover(); + cy.get(".ant-tooltip-inner").should("contain", "December"); + }); + // description + cy.getByTestId(`monitor-result-${webMonitorKey}`).should( + "contain", + "92 Browser Requests, 5 Cookies detected.", + ); + // monitor name + cy.getByTestId(`monitor-result-${webMonitorKey}`).should( + "contain", + "my web monitor 2", + ); + }); + it("should have appropriate actions for web monitors", () => { + cy.wait("@getMonitorResults"); + // Add button + // TODO: [HJ-337] uncomment when Add button is implemented + // cy.getByTestId(`add-button-${webMonitorKey}`).should("exist"); + // Review button + cy.getByTestId(`review-button-${webMonitorKey}`).should( + "have.attr", + "href", + `${ACTION_CENTER_ROUTE}/${webMonitorKey}`, + ); + }); + it.skip("Should have appropriate actions for Integrations monitors", () => { + cy.wait("@getMonitorResults"); + // Classify button + cy.getByTestId(`review-button-${integrationMonitorKey}`).should( + "have.attr", + "href", + `${ACTION_CENTER_ROUTE}/${integrationMonitorKey}`, + ); + // Ignore button + cy.getByTestId(`ignore-button-${integrationMonitorKey}`).should("exist"); + }); + it.skip("Should have appropriate actions for SSO monitors", () => { + cy.wait("@getMonitorResults"); + // Add button + cy.getByTestId(`add-button-${webMonitorKey}`).should("exist"); + // Ignore button + cy.getByTestId(`ignore-button-${webMonitorKey}`).should("exist"); + }); + it.skip("Should paginate results", () => { + // TODO: mock pagination and also test skeleton loading state + }); + }); + + describe("Action center system aggregate results", () => { + const webMonitorKey = "my_web_monitor_1"; + beforeEach(() => { + cy.visit(`${ACTION_CENTER_ROUTE}/${webMonitorKey}`); + }); + it("should display a breadcrumb", () => { + cy.getByTestId("page-breadcrumb").within(() => { + cy.get("a.ant-breadcrumb-link") + .should("contain", "All activity") + .should("have.attr", "href", ACTION_CENTER_ROUTE); + cy.contains("my_web_monitor_1").should("exist"); + }); + }); + it("should render the aggregated system results in a table", () => { + cy.wait("@getSystemAggregateResults"); + cy.getByTestId("column-system_name").should("exist"); + cy.getByTestId("column-total_updates").should("exist"); + cy.getByTestId("column-data_use").should("exist"); + cy.getByTestId("column-locations").should("exist"); + cy.getByTestId("column-domains").should("exist"); + cy.getByTestId("column-actions").should("exist"); + cy.getByTestId("search-bar").should("exist"); + cy.getByTestId("pagination-btn").should("exist"); + cy.getByTestId("row-0-col-system_name").within(() => { + cy.getByTestId("change-icon").should("exist"); // new result + cy.contains("Uncategorized assets").should("exist"); + }); + // data use column should be empty for uncategorized assets + cy.getByTestId("row-0-col-data_use").children().should("have.length", 0); + cy.getByTestId("row-1-col-system_name").within(() => { + cy.getByTestId("change-icon").should("not.exist"); // existing result + cy.contains("Google Tag Manager").should("exist"); + }); + // TODO: data use column should not be empty for other assets + // cy.getByTestId("row-1-col-data_use").children().should("not.have.length", 0); + + // multiple locations + cy.getByTestId("row-2-col-locations").should("contain", "2 locations"); + // single location + cy.getByTestId("row-3-col-locations").should("contain", "USA"); + + // multiple domains + cy.getByTestId("row-0-col-domains").should("contain", "29 domains"); + // single domain + cy.getByTestId("row-3-col-domains").should( + "contain", + "analytics.google.com", + ); + }); + // it("should navigate to table view on row click", () => { + // cy.getByTestId("row-1").click(); + // cy.url().should("contain", "fds.1046"); + // cy.getByTestId("page-breadcrumb").should("contain", "fds.1046"); + // }); + }); +}); diff --git a/clients/admin-ui/cypress/e2e/datamap-report.cy.ts b/clients/admin-ui/cypress/e2e/datamap-report.cy.ts index 7d15b5a230..40c96cbe54 100644 --- a/clients/admin-ui/cypress/e2e/datamap-report.cy.ts +++ b/clients/admin-ui/cypress/e2e/datamap-report.cy.ts @@ -323,6 +323,15 @@ describe("Data map report table", () => { it("should filter the table by making a selection", () => { cy.getByTestId("filter-multiple-systems-btn").click(); cy.getByTestId("datamap-report-filter-modal").should("be.visible"); + cy.getByTestId("filter-modal-accordion-button") + .eq(0) + .should("have.text", "Data use"); + cy.getByTestId("filter-modal-accordion-button") + .eq(1) + .should("have.text", "Data categories"); + cy.getByTestId("filter-modal-accordion-button") + .eq(2) + .should("have.text", "Data subject"); cy.getByTestId("filter-modal-accordion-button").eq(1).click(); cy.getByTestId("filter-modal-checkbox-tree-categories").should( "be.visible", @@ -389,14 +398,15 @@ describe("Data map report table", () => { cy.get("#toast-datamap-report-toast") .should("be.visible") .should("have.attr", "data-status", "success"); - cy.getByTestId("custom-reports-trigger") - .should("contain.text", "My Custom Report") - .click(); + cy.getByTestId("custom-reports-trigger").should( + "contain.text", + "My Custom Report", + ); cy.getByTestId("fidesTable").within(() => { // reordering applied to report cy.get("thead th").eq(2).should("contain.text", "Legal name"); // column visibility applied to report - cy.get("thead th").eq(4).should("not.contain.text", "Data subject"); + cy.getByTestId("column-data_subjects").should("not.exist"); }); cy.getByTestId("group-by-menu").should( "contain.text", @@ -442,10 +452,36 @@ describe("Data map report table", () => { cy.getByTestId("custom-reports-reset-button").click(); cy.getByTestId("apply-report-button").click(); cy.getByTestId("custom-reports-popover").should("not.be.visible"); + cy.getByTestId("custom-reports-trigger").should( "contain.text", "Reports", ); + cy.getByTestId("fidesTable").within(() => { + // reordering reverted + cy.get("thead th").eq(2).should("contain.text", "Data categories"); + // column visibility restored + cy.getByTestId("column-data_subjects").should("exist"); + }); + cy.getByTestId("group-by-menu").should("contain.text", "Group by system"); + cy.getByTestId("more-menu").click(); + cy.getByTestId("edit-columns-btn").click(); + cy.get("button#data_subjects").should( + "have.attr", + "aria-checked", + "true", + ); + cy.getByTestId("column-settings-close-button").click(); + cy.getByTestId("filter-multiple-systems-btn").click(); + cy.getByTestId("datamap-report-filter-modal") + .should("be.visible") + .within(() => { + cy.getByTestId("filter-modal-accordion-button").eq(0).click(); + cy.getByTestId("checkbox-Analytics").within(() => { + cy.get("[data-checked]").should("not.exist"); + }); + cy.getByTestId("standard-dialog-close-btn").click(); + }); }); it("should allow the user cancel a report selection", () => { cy.wait("@getCustomReportsMinimal"); diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/aggregate-results.json b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/aggregate-results.json new file mode 100644 index 0000000000..0a870f84e1 --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/aggregate-results.json @@ -0,0 +1,40 @@ +{ + "items": [ + { + "name": "my web monitor 2", + "key": "my_web_monitor_2", + "last_monitored": "2024-12-17T17:31:20.791014Z", + "updates": { + "Browser Request": 92, + "Cookie": 5 + }, + "total_updates": 97 + }, + { + "name": "my web monitor 1", + "key": "my_web_monitor_1", + "last_monitored": "2024-12-17T17:31:02.319068Z", + "updates": { + "Browser Request": 201, + "Cookie": 24 + }, + "total_updates": 225 + }, + { + "name": "My New BQ Monitor", + "key": "My_New_BQ_Monitor", + "last_monitored": "2024-12-16T20:04:16.824025Z", + "updates": { + "Database": 2, + "Field": 216, + "Schema": 13, + "Table": 22 + }, + "total_updates": 253 + } + ], + "total": 3, + "page": 1, + "size": 25, + "pages": 1 +} diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json new file mode 100644 index 0000000000..3bcc4b2328 --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json @@ -0,0 +1,84 @@ +{ + "items": [ + { + "id": null, + "name": null, + "system_key": null, + "vendor_id": null, + "total_updates": 108, + "locations": ["USA"], + "domains": [ + "alb.reddit.com", + "api.hubapi.com", + "app.revenuehero.io", + ".ethyca.com", + "ethyca.com", + "ethyca.fides-cdn.ethyca.com", + "forms.hscollectedforms.net", + "forms.hubspot.com", + "forms-na1.hsforms.com", + "googleads.g.doubleclick.net", + ".hsadspixel.net", + ".hsforms.com", + ".hs-scripts.com", + ".hubspot.com", + "js.hsadspixel.net", + "js.hs-analytics.net", + "js.hs-banner.com", + "js.hscollectedforms.net", + "js.hs-scripts.com", + "kit.fontawesome.com", + ".linkedin.com", + "pixel-config.reddit.com", + "px.ads.linkedin.com", + "snap.licdn.com", + "stats.g.doubleclick.net", + "track.hubspot.com", + "www.clickcease.com", + ".www.linkedin.com", + "www.redditstatic.com" + ] + }, + { + "id": "system_key-72649f03-7a30-4758-9772-e74fca3b6788", + "name": "Google Tag Manager", + "system_key": "system_key-72649f03-7a30-4758-9772-e74fca3b6788", + "vendor_id": "fds.1046", + "total_updates": 10, + "locations": ["USA"], + "domains": [ + "td.doubleclick.net", + "www.google.com", + "www.googletagmanager.com" + ] + }, + { + "id": "system_key-652c8984-ade7-470b-bce4-7e184621be9d", + "name": "Hubspot", + "system_key": "system_key-652c8984-ade7-470b-bce4-7e184621be9d", + "vendor_id": "fds.1053", + "total_updates": 6, + "locations": ["USA", "Canada"], + "domains": [ + "forms.hsforms.com", + ".hs-analytics.net", + ".hs-banner.com", + ".hsforms.net", + "js.hsforms.net" + ] + }, + { + "id": "fds.1047", + "name": "Google Analytics", + "system_key": null, + "vendor_id": "fds.1047", + "total_updates": 1, + "locations": ["USA"], + "domains": ["analytics.google.com"] + } + ], + "total": 4, + "page": 1, + "size": 25, + "pages": 1 +} diff --git a/clients/admin-ui/cypress/support/stubs.ts b/clients/admin-ui/cypress/support/stubs.ts index bb2e40537c..6c3109f1e9 100644 --- a/clients/admin-ui/cypress/support/stubs.ts +++ b/clients/admin-ui/cypress/support/stubs.ts @@ -502,3 +502,23 @@ export const stubFidesCloud = () => { domain_verification_records: [], }).as("getFidesCloud"); }; + +export const stubActionCenter = () => { + cy.intercept("GET", "/api/v1/config*", { + body: { + detection_discovery: { + website_monitor_enabled: true, + }, + }, + }).as("getTranslationConfig"); + cy.intercept("GET", "/api/v1/plus/discovery-monitor/aggregate-results*", { + fixture: "detection-discovery/activity-center/aggregate-results", + }).as("getMonitorResults"); + cy.intercept( + "GET", + "/api/v1//plus/discovery-monitor/system-aggregate-results*", + { + fixture: "detection-discovery/activity-center/system-aggregate-results", + }, + ).as("getSystemAggregateResults"); +}; diff --git a/clients/admin-ui/package.json b/clients/admin-ui/package.json index 1fd75f5903..34dd8bbb4d 100644 --- a/clients/admin-ui/package.json +++ b/clients/admin-ui/package.json @@ -30,6 +30,7 @@ }, "dependencies": { "@ant-design/cssinjs": "^1.21.0", + "@date-fns/tz": "^1.2.0", "@fontsource/inter": "^4.5.15", "@monaco-editor/react": "^4.6.0", "@reduxjs/toolkit": "^1.9.3", @@ -40,8 +41,8 @@ "cytoscape": "^3.30.0", "cytoscape-klay": "^3.1.4", "d3-hierarchy": "^3.1.2", - "date-fns": "^2.29.3", - "date-fns-tz": "^2.0.0", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", "eslint-plugin-tailwindcss": "^3.17.4", "fides-js": "^0.0.1", "fidesui": "*", diff --git a/clients/admin-ui/src/features/common/SearchBar.tsx b/clients/admin-ui/src/features/common/SearchBar.tsx index 973ca254ed..248d34e44b 100644 --- a/clients/admin-ui/src/features/common/SearchBar.tsx +++ b/clients/admin-ui/src/features/common/SearchBar.tsx @@ -24,7 +24,7 @@ const SearchBar = ({ onChange(event.target.value); return ( - + ( - - {children} - -); - -export { Layout }; diff --git a/clients/admin-ui/src/features/common/custom-fields/index.ts b/clients/admin-ui/src/features/common/custom-fields/index.ts index 90342f6b0e..c1f980165d 100644 --- a/clients/admin-ui/src/features/common/custom-fields/index.ts +++ b/clients/admin-ui/src/features/common/custom-fields/index.ts @@ -2,5 +2,4 @@ export * from "./constants"; export * from "./CustomFieldsList"; export * from "./helpers"; export * from "./hooks"; -export * from "./Layout"; export * from "./types"; diff --git a/clients/admin-ui/src/features/common/nav/v2/nav-config.ts b/clients/admin-ui/src/features/common/nav/v2/nav-config.ts index 4b5fba1402..ed8ae94d28 100644 --- a/clients/admin-ui/src/features/common/nav/v2/nav-config.ts +++ b/clients/admin-ui/src/features/common/nav/v2/nav-config.ts @@ -38,6 +38,13 @@ export const NAV_CONFIG: NavConfigGroup[] = [ { title: "Detection & Discovery", routes: [ + { + title: "Action center", + path: routes.ACTION_CENTER_ROUTE, + scopes: [], + requiresFlag: "webMonitor", + requiresPlus: true, + }, { title: "Activity", path: routes.DETECTION_DISCOVERY_ACTIVITY_ROUTE, diff --git a/clients/admin-ui/src/features/common/nav/v2/routes.ts b/clients/admin-ui/src/features/common/nav/v2/routes.ts index b043b16fc9..2286ce49a0 100644 --- a/clients/admin-ui/src/features/common/nav/v2/routes.ts +++ b/clients/admin-ui/src/features/common/nav/v2/routes.ts @@ -20,6 +20,7 @@ export const DATASET_COLLECTION_SUBFIELD_DETAIL_ROUTE = "/dataset/[datasetId]/[collectionName]/[...subfieldNames]"; // Detection and discovery +export const ACTION_CENTER_ROUTE = "/data-discovery/action-center"; export const DETECTION_DISCOVERY_ACTIVITY_ROUTE = "/data-discovery/activity"; export const DATA_DETECTION_ROUTE = "/data-discovery/detection"; export const DATA_DETECTION_ROUTE_DETAIL = diff --git a/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx b/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx index 49623716db..457e6fbfdc 100644 --- a/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx +++ b/clients/admin-ui/src/features/common/table/v2/PaginationBar.tsx @@ -44,7 +44,7 @@ export const useServerSidePagination = () => { const defaultPageIndex = 1; const [pageSize, setPageSize] = useState(PAGE_SIZES[0]); const [pageIndex, setPageIndex] = useState(defaultPageIndex); - const [totalPages, setTotalPages] = useState(); + const [totalPages, setTotalPages] = useState(1); const onPreviousPageClick = useCallback(() => { setPageIndex((prev) => prev - 1); }, [setPageIndex]); @@ -53,7 +53,7 @@ export const useServerSidePagination = () => { setPageIndex((prev) => prev + 1); }, [setPageIndex]); const isNextPageDisabled = useMemo( - () => pageIndex === totalPages, + () => !!totalPages && (pageIndex === totalPages || totalPages < 2), [pageIndex, totalPages], ); diff --git a/clients/admin-ui/src/features/common/utils.ts b/clients/admin-ui/src/features/common/utils.ts index acc4d86588..ab18bc6803 100644 --- a/clients/admin-ui/src/features/common/utils.ts +++ b/clients/admin-ui/src/features/common/utils.ts @@ -32,7 +32,7 @@ export const debounce = (fn: (props?: any) => void, ms = 0) => { }; export const formatDate = (value: string | number | Date): string => - format(new Date(value), "MMMM d, Y, KK:mm:ss z"); + format(new Date(value), "MMMM d, y, KK:mm:ss z"); export const utf8ToB64 = (str: string): string => window.btoa(unescape(encodeURIComponent(str))); @@ -116,3 +116,7 @@ export const getOptionsFromMap = ( label: value, value: key, })); + +export const getWebsiteIconUrl = (hostname: string) => { + return `https://icons.duckduckgo.com/ip3/${hostname}.ico`; +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorsPage.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorsPage.tsx new file mode 100644 index 0000000000..cb80d6c6ba --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorsPage.tsx @@ -0,0 +1,28 @@ +import { AntAlert as Alert, AntFlex as Flex, Spinner } from "fidesui"; + +import Layout from "~/features/common/Layout"; + +interface DisabledMonitorsPageProps { + isConfigLoading: boolean; +} + +const DISABLED_MONITORS_MESSAGE = "Action center is currently disabled."; + +export const DisabledMonitorsPage = ({ + isConfigLoading, +}: DisabledMonitorsPageProps) => ( + + + {isConfigLoading ? ( + + ) : ( + + )} + + +); diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorsResult.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorsResult.tsx new file mode 100644 index 0000000000..dfd82237fb --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorsResult.tsx @@ -0,0 +1,15 @@ +import { AntButton as Button, AntEmpty as Empty } from "fidesui"; +import NextLink from "next/link"; + +import { INTEGRATION_MANAGEMENT_ROUTE } from "~/features/common/nav/v2/routes"; + +export const EmptyMonitorsResult = () => ( + + + + + +); diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx new file mode 100644 index 0000000000..3614217ffe --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx @@ -0,0 +1,100 @@ +import { formatDistance } from "date-fns"; +import { + AntAvatar as Avatar, + AntFlex as Flex, + AntList as List, + AntListItemProps as ListItemProps, + AntSkeleton as Skeleton, + AntTooltip as Tooltip, + AntTypography as Typography, + Icons, +} from "fidesui"; +import NextLink from "next/link"; +import { useEffect, useState } from "react"; + +import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; +import { formatDate, getWebsiteIconUrl } from "~/features/common/utils"; + +import { MonitorAggregatedResults } from "./types"; + +const { Text } = Typography; + +interface MonitorResultProps extends ListItemProps { + monitorSummary: MonitorAggregatedResults; + showSkeleton?: boolean; +} + +export const MonitorResult = ({ + monitorSummary, + showSkeleton, + ...props +}: MonitorResultProps) => { + const [iconUrl, setIconUrl] = useState(undefined); + + const { + name, + property, + total_updates: totalUpdates, + updates, + last_monitored: lastMonitored, + warning, + key, + } = monitorSummary; + + const assetCountString = Object.entries(updates) + .map((update) => { + return `${update[1]} ${update[0]}s`; + }) + .join(", "); + + const lastMonitoredDistance = lastMonitored + ? formatDistance(new Date(lastMonitored), new Date(), { + addSuffix: true, + }) + : undefined; + + useEffect(() => { + if (property) { + setIconUrl(getWebsiteIconUrl(property)); + } + }, [property]); + + return ( + + + } + title={ + + {`${totalUpdates} assets detected${property ? `on ${property}` : ""}`} + {!!warning && ( + + + + )} + + } + description={`${assetCountString} detected.`} + /> + + + {name} + + {!!lastMonitoredDistance && ( + + {lastMonitoredDistance} + + )} + + + + ); +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts new file mode 100644 index 0000000000..5e25656721 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts @@ -0,0 +1,48 @@ +import { baseApi } from "~/features/common/api.slice"; +import { PaginationQueryParams } from "~/types/common/PaginationQueryParams"; + +import { + MonitorSummaryPaginatedResponse, + MonitorSystemAggregatePaginatedResponse, +} from "./types"; + +const actionCenterApi = baseApi.injectEndpoints({ + endpoints: (build) => ({ + getAggregateMonitorResults: build.query< + MonitorSummaryPaginatedResponse, + { + search?: string; + } & PaginationQueryParams + >({ + query: ({ page = 1, size = 20, search }) => ({ + url: `/plus/discovery-monitor/aggregate-results`, + params: { page, size, search, diff_status: "addition" }, + }), + providesTags: ["Discovery Monitor Results"], + }), + getDiscoveredSystemAggregate: build.query< + MonitorSystemAggregatePaginatedResponse, + { + key: string; + search?: string; + } & PaginationQueryParams + >({ + query: ({ key, page = 1, size = 20, search }) => ({ + url: `/plus/discovery-monitor/system-aggregate-results`, + params: { + monitor_config_id: key, + page, + size, + search, + diff_status: "addition", + }, + }), + providesTags: ["Discovery Monitor Results"], + }), + }), +}); + +export const { + useGetAggregateMonitorResultsQuery, + useGetDiscoveredSystemAggregateQuery, +} = actionCenterApi; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx new file mode 100644 index 0000000000..747ab1ef6c --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx @@ -0,0 +1,75 @@ +import { createColumnHelper } from "@tanstack/react-table"; + +import { DefaultCell } from "~/features/common/table/v2"; + +import { DiscoveredSystemActionsCell } from "../tables/cells/DiscoveredSystemAggregateActionsCell"; +import { DiscoveredSystemStatusCell } from "../tables/cells/DiscoveredSystemAggregateStatusCell"; +import { MonitorSystemAggregate } from "../types"; + +export const useDiscoveredSystemAggregateColumns = () => { + const columnHelper = createColumnHelper(); + + const columns = [ + columnHelper.accessor((row) => row.name, { + id: "system_name", + cell: (props) => ( + + ), + header: "System", + meta: { + width: "auto", + }, + }), + columnHelper.accessor((row) => row.total_updates, { + id: "total_updates", + cell: (props) => , + header: "Assets", + size: 80, + }), + columnHelper.display({ + id: "data_use", + header: "Categories of consent", + meta: { + width: "auto", + }, + }), + columnHelper.accessor((row) => row.locations, { + id: "locations", + cell: (props) => ( + 1 + ? `${props.getValue().length} locations` + : props.getValue()[0] + } + /> + ), + header: "Locations", + }), + columnHelper.accessor((row) => row.domains, { + id: "domains", + cell: (props) => ( + 1 + ? `${props.getValue().length} domains` + : props.getValue()[0] + } + /> + ), + header: "Domains", + }), + columnHelper.display({ + id: "actions", + cell: (props) => ( + + ), + header: "Actions", + meta: { + width: "auto", + }, + }), + ]; + + return { columns }; +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx new file mode 100644 index 0000000000..01d4b8c4d5 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx @@ -0,0 +1,101 @@ +import { getCoreRowModel, useReactTable } from "@tanstack/react-table"; +import { Box, Flex } from "fidesui"; +import { useEffect, useState } from "react"; + +import { + FidesTableV2, + PaginationBar, + TableActionBar, + TableSkeletonLoader, + useServerSidePagination, +} from "~/features/common/table/v2"; +import { useGetDiscoveredSystemAggregateQuery } from "~/features/data-discovery-and-detection/action-center/action-center.slice"; + +import { SearchInput } from "../../SearchInput"; +import { useDiscoveredSystemAggregateColumns } from "../hooks/useDiscoveredSystemAggregateColumns"; + +interface DiscoveredSystemAggregateTableProps { + monitorId: string; +} + +export const DiscoveredSystemAggregateTable = ({ + monitorId, +}: DiscoveredSystemAggregateTableProps) => { + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + resetPageIndexToDefault, + } = useServerSidePagination(); + const [searchQuery, setSearchQuery] = useState(""); + + useEffect(() => { + resetPageIndexToDefault(); + }, [monitorId, searchQuery, resetPageIndexToDefault]); + + const { data, isLoading, isFetching } = useGetDiscoveredSystemAggregateQuery({ + key: monitorId, + page: pageIndex, + size: pageSize, + search: searchQuery, + }); + + useEffect(() => { + if (data) { + setTotalPages(data.pages || 1); + } + }, [data, setTotalPages]); + + const { columns } = useDiscoveredSystemAggregateColumns(); + + const tableInstance = useReactTable({ + getCoreRowModel: getCoreRowModel(), + columns, + manualPagination: true, + data: data?.items || [], + columnResizeMode: "onChange", + }); + + if (isLoading) { + return ; + } + + return ( + <> + + + + + + + + + + + + + ); +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx new file mode 100644 index 0000000000..d00127b884 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx @@ -0,0 +1,14 @@ +import { AntFlex as Flex } from "fidesui"; + +import { MonitorSystemAggregate } from "../../types"; + +interface DiscoveredSystemActionsCellProps { + system: MonitorSystemAggregate; +} + +export const DiscoveredSystemActionsCell = ({ + system, +}: DiscoveredSystemActionsCellProps) => { + console.log(system); + return ; +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx new file mode 100644 index 0000000000..3c13f34290 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx @@ -0,0 +1,33 @@ +import { Flex, Text, Tooltip } from "fidesui"; + +import { STATUS_INDICATOR_MAP } from "~/features/data-discovery-and-detection/statusIndicators"; + +import { MonitorSystemAggregate } from "../../types"; + +interface DiscoveredSystemStatusCellProps { + system: MonitorSystemAggregate; +} + +export const DiscoveredSystemStatusCell = ({ + system, +}: DiscoveredSystemStatusCellProps) => { + return ( + + {!system?.system_key && ( + + {/* icon has to be wrapped in a span for the tooltip to work */} + {STATUS_INDICATOR_MAP.Change} + + )} + + {system?.name || "Uncategorized assets"} + + + ); +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts new file mode 100644 index 0000000000..f2933bca51 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts @@ -0,0 +1,27 @@ +import { PaginatedResponse } from "~/types/common/PaginationQueryParams"; + +// TODO: [HJ-334] remove these in favor of autogenerated types from the API +export interface MonitorAggregatedResults { + updates: Record; + property?: string; // this is a guess, it doesn't exist yet in the API + last_monitored: string | number; + key: string; + name: string; + total_updates: number; + warning?: boolean | string; +} + +export interface MonitorSummaryPaginatedResponse + extends PaginatedResponse {} + +export interface MonitorSystemAggregate { + name: string; + system_key: string | null; // null when the system is not a known system + vendor_id: string; + total_updates: 0; + locations: string[]; + domains: string[]; +} + +export interface MonitorSystemAggregatePaginatedResponse + extends PaginatedResponse {} diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx index fc67fb016f..1bded963b2 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx @@ -148,7 +148,7 @@ export const DatamapReportFilterModal = ({ data-testid="datamap-report-filter-modal" > - + { ], ); - useEffect(() => { - if (datamapReport?.items?.length) { - const columnIDs = Object.keys(datamapReport.items[0]); - setColumnOrder(getColumnOrder(groupBy, columnIDs)); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [groupBy, datamapReport]); - const { isOpen: isColumnSettingsOpen, onOpen: onColumnSettingsOpen, @@ -306,6 +302,20 @@ export const DatamapReportTable = () => { }, }); + useEffect(() => { + if (groupBy && !!tableInstance) { + if (tableInstance.getState().columnOrder.length === 0) { + const tableColumnIds = tableInstance.getAllColumns().map((c) => c.id); + setColumnOrder(getColumnOrder(groupBy, tableColumnIds)); + } else { + setColumnOrder( + getColumnOrder(groupBy, tableInstance.getState().columnOrder), + ); + } + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [groupBy, tableInstance]); + useEffect(() => { // changing the groupBy should wait until the data is loaded to update the grouping const newGrouping = getGrouping(groupBy); @@ -345,12 +355,41 @@ export const DatamapReportTable = () => { const handleSavedReport = ( savedReport: CustomReportResponse | null, - resetForm: ( + resetColumnNameForm: ( nextState?: Partial>> | undefined, ) => void, ) => { + if (!savedReport && !savedCustomReportId) { + return; + } if (!savedReport) { - setSavedCustomReportId(""); + try { + setSavedCustomReportId(""); + + /* NOTE: we can't just use tableInstance.reset() here because it will reset the table to the initial state, which is likely to include report settings that were saved in the user's local storage. Instead, we need to reset each individual setting to its default value. */ + + // reset column visibility (must happen before updating order) + setColumnVisibility(DEFAULT_COLUMN_VISIBILITY); + tableInstance.toggleAllColumnsVisible(true); + tableInstance.setColumnVisibility(DEFAULT_COLUMN_VISIBILITY); + + // reset column order (must happen prior to updating groupBy) + setColumnOrder([]); + tableInstance.setColumnOrder([]); + + // reset groupBy and filters (will automatically update the tableinstance) + setGroupBy(DATAMAP_GROUPING.SYSTEM_DATA_USE); + setSelectedFilters(DEFAULT_COLUMN_FILTERS); + + // reset column names + setColumnNameMapOverrides({}); + resetColumnNameForm({ values: {} }); + } catch (error: any) { + toast({ + status: "error", + description: "There was a problem resetting the report.", + }); + } return; } try { @@ -369,8 +408,8 @@ export const DatamapReportTable = () => { ); if (savedGroupBy) { + // No need to manually update the tableInstance here; setting the groupBy will trigger the useEffect to update the grouping. setGroupBy(savedGroupBy); - tableInstance.setGrouping(getGrouping(savedGroupBy)); } if (savedFilters) { setSelectedFilters(savedFilters); @@ -394,7 +433,7 @@ export const DatamapReportTable = () => { }, ); setColumnNameMapOverrides(columnNameMap); - resetForm({ values: columnNameMap }); + resetColumnNameForm({ values: columnNameMap }); } setSavedCustomReportId(savedReport.id); toast({ diff --git a/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx b/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx index 3051bfd243..883f331fdf 100644 --- a/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx @@ -12,6 +12,17 @@ import { DATAMAP_GROUPING } from "~/types/api"; import { DatamapReportFilterSelections } from "../types"; import { COLUMN_IDS, DATAMAP_LOCAL_STORAGE_KEYS } from "./constants"; +export const DEFAULT_COLUMN_VISIBILITY = { + [COLUMN_IDS.SYSTEM_UNDECLARED_DATA_CATEGORIES]: false, + [COLUMN_IDS.DATA_USE_UNDECLARED_DATA_CATEGORIES]: false, +}; + +export const DEFAULT_COLUMN_FILTERS = { + dataUses: [], + dataSubjects: [], + dataCategories: [], +}; + interface DatamapReportContextProps { savedCustomReportId: string; setSavedCustomReportId: Dispatch>; @@ -51,11 +62,7 @@ export const DatamapReportProvider = ({ const [selectedFilters, setSelectedFilters] = useLocalStorage( DATAMAP_LOCAL_STORAGE_KEYS.FILTERS, - { - dataUses: [], - dataSubjects: [], - dataCategories: [], - }, + DEFAULT_COLUMN_FILTERS, ); const [columnOrder, setColumnOrder] = useLocalStorage( @@ -65,10 +72,7 @@ export const DatamapReportProvider = ({ const [columnVisibility, setColumnVisibility] = useLocalStorage< Record - >(DATAMAP_LOCAL_STORAGE_KEYS.COLUMN_VISIBILITY, { - [COLUMN_IDS.SYSTEM_UNDECLARED_DATA_CATEGORIES]: false, - [COLUMN_IDS.DATA_USE_UNDECLARED_DATA_CATEGORIES]: false, - }); + >(DATAMAP_LOCAL_STORAGE_KEYS.COLUMN_VISIBILITY, DEFAULT_COLUMN_VISIBILITY); const [columnSizing, setColumnSizing] = useLocalStorage< Record diff --git a/clients/admin-ui/src/features/datamap/reporting/utils.ts b/clients/admin-ui/src/features/datamap/reporting/utils.ts index 6c7b910134..343dce7969 100644 --- a/clients/admin-ui/src/features/datamap/reporting/utils.ts +++ b/clients/admin-ui/src/features/datamap/reporting/utils.ts @@ -12,10 +12,7 @@ export const getGrouping = (groupBy?: DATAMAP_GROUPING) => { } }; -export const getColumnOrder = ( - groupBy: DATAMAP_GROUPING, - columnIDs: string[], -) => { +export const getPrefixColumns = (groupBy: DATAMAP_GROUPING) => { let columnOrder: string[] = []; if (DATAMAP_GROUPING.SYSTEM_DATA_USE === groupBy) { columnOrder = [COLUMN_IDS.SYSTEM_NAME, COLUMN_IDS.DATA_USE]; @@ -23,6 +20,14 @@ export const getColumnOrder = ( if (DATAMAP_GROUPING.DATA_USE_SYSTEM === groupBy) { columnOrder = [COLUMN_IDS.DATA_USE, COLUMN_IDS.SYSTEM_NAME]; } + return columnOrder; +}; + +export const getColumnOrder = ( + groupBy: DATAMAP_GROUPING, + columnIDs: string[], +) => { + let columnOrder: string[] = getPrefixColumns(groupBy); columnOrder = columnOrder.concat( columnIDs.filter( (columnID) => @@ -31,14 +36,3 @@ export const getColumnOrder = ( ); return columnOrder; }; - -export const getPrefixColumns = (groupBy: DATAMAP_GROUPING) => { - let columnOrder: string[] = []; - if (DATAMAP_GROUPING.SYSTEM_DATA_USE === groupBy) { - columnOrder = [COLUMN_IDS.SYSTEM_NAME, COLUMN_IDS.DATA_USE]; - } - if (DATAMAP_GROUPING.DATA_USE_SYSTEM === groupBy) { - columnOrder = [COLUMN_IDS.DATA_USE, COLUMN_IDS.SYSTEM_NAME]; - } - return columnOrder; -}; diff --git a/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx b/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx index fc51a7ae17..86b15a4645 100644 --- a/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx +++ b/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx @@ -12,12 +12,19 @@ import { CONNECTOR_LOGOS_PATH, FALLBACK_CONNECTOR_LOGOS_PATH, } from "./constants"; -import { isConnectionSystemTypeMap, isDatastoreConnection } from "./types"; type ConnectionTypeLogoProps = { data: string | ConnectionConfigurationResponse | ConnectionSystemTypeMap; }; +const isDatastoreConnection = ( + obj: any, +): obj is ConnectionConfigurationResponse => + (obj as ConnectionConfigurationResponse).connection_type !== undefined; + +const isConnectionSystemTypeMap = (obj: any): obj is ConnectionSystemTypeMap => + (obj as ConnectionSystemTypeMap).encoded_icon !== undefined; + const ConnectionTypeLogo = ({ data, ...props diff --git a/clients/admin-ui/src/features/datastore-connections/types.ts b/clients/admin-ui/src/features/datastore-connections/types.d.ts similarity index 91% rename from clients/admin-ui/src/features/datastore-connections/types.ts rename to clients/admin-ui/src/features/datastore-connections/types.d.ts index e4a171038f..7d1e20d841 100644 --- a/clients/admin-ui/src/features/datastore-connections/types.ts +++ b/clients/admin-ui/src/features/datastore-connections/types.d.ts @@ -1,6 +1,5 @@ import { ConnectionConfigurationResponse, - ConnectionSystemTypeMap, ConnectionType, DatasetConfigCtlDataset, SystemType, @@ -128,16 +127,6 @@ export type DatastoreConnectionResponse = { ]; }; -export const isDatastoreConnection = ( - obj: any, -): obj is ConnectionConfigurationResponse => - (obj as ConnectionConfigurationResponse).connection_type !== undefined; - -export const isConnectionSystemTypeMap = ( - obj: any, -): obj is ConnectionSystemTypeMap => - (obj as ConnectionSystemTypeMap).encoded_icon !== undefined; - export type DatastoreConnectionParams = { search: string; connection_type?: string[]; diff --git a/clients/admin-ui/src/features/locations/LocationManagement.tsx b/clients/admin-ui/src/features/locations/LocationManagement.tsx index 40931434d1..ac244ebf8e 100644 --- a/clients/admin-ui/src/features/locations/LocationManagement.tsx +++ b/clients/admin-ui/src/features/locations/LocationManagement.tsx @@ -98,7 +98,6 @@ const LocationManagement = ({ data }: { data: LocationRegulationResponse }) => { placeholder="Search" search={search} onClear={() => setSearch("")} - data-testid="search-bar" /> diff --git a/clients/admin-ui/src/features/locations/RegulationManagement.tsx b/clients/admin-ui/src/features/locations/RegulationManagement.tsx index 795d2b77ff..980801e3fc 100644 --- a/clients/admin-ui/src/features/locations/RegulationManagement.tsx +++ b/clients/admin-ui/src/features/locations/RegulationManagement.tsx @@ -103,7 +103,6 @@ const RegulationManagement = ({ placeholder="Search" search={search} onClear={() => setSearch("")} - data-testid="search-bar" /> diff --git a/clients/admin-ui/src/flags.json b/clients/admin-ui/src/flags.json index f33a2e81ee..5f2b32cea2 100644 --- a/clients/admin-ui/src/flags.json +++ b/clients/admin-ui/src/flags.json @@ -36,6 +36,12 @@ "test": true, "production": false }, + "webMonitor": { + "description": "Monitor websites for activity", + "development": true, + "test": true, + "production": false + }, "ssoAuthentication": { "description": "SSO Authentication Providers (OpenID)", "development": true, diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx new file mode 100644 index 0000000000..5f96ddfa1a --- /dev/null +++ b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx @@ -0,0 +1,27 @@ +import { NextPage } from "next"; +import { useRouter } from "next/router"; + +import FixedLayout from "~/features/common/FixedLayout"; +import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { DiscoveredSystemAggregateTable } from "~/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable"; + +const MonitorResultSystems: NextPage = () => { + const router = useRouter(); + const monitorId = decodeURIComponent(router.query.monitorId as string); + + return ( + + + + + ); +}; + +export default MonitorResultSystems; diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx new file mode 100644 index 0000000000..598a357f4d --- /dev/null +++ b/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx @@ -0,0 +1,181 @@ +import { + AntButton as Button, + AntDivider as Divider, + AntFlex as Flex, + AntList as List, + useToast, +} from "fidesui"; +import NextLink from "next/link"; +import { useCallback, useEffect, useState } from "react"; + +import Layout from "~/features/common/Layout"; +import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { + PaginationBar, + useServerSidePagination, +} from "~/features/common/table/v2"; +import { useGetAggregateMonitorResultsQuery } from "~/features/data-discovery-and-detection/action-center/action-center.slice"; +import { DisabledMonitorsPage } from "~/features/data-discovery-and-detection/action-center/DisabledMonitorsPage"; +import { EmptyMonitorsResult } from "~/features/data-discovery-and-detection/action-center/EmptyMonitorsResult"; +import { MonitorResult } from "~/features/data-discovery-and-detection/action-center/MonitorResult"; +import { MonitorAggregatedResults } from "~/features/data-discovery-and-detection/action-center/types"; +import { SearchInput } from "~/features/data-discovery-and-detection/SearchInput"; +import { useGetConfigurationSettingsQuery } from "~/features/privacy-requests"; + +const ActionCenterPage = () => { + const toast = useToast(); + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + resetPageIndexToDefault, + } = useServerSidePagination(); + const [searchQuery, setSearchQuery] = useState(""); + const { data: appConfig, isLoading: isConfigLoading } = + useGetConfigurationSettingsQuery({ + api_set: false, + }); + const webMonitorEnabled = + !!appConfig?.detection_discovery?.website_monitor_enabled; + + useEffect(() => { + resetPageIndexToDefault(); + }, [searchQuery, resetPageIndexToDefault]); + + const { data, isError, isLoading, isFetching } = + useGetAggregateMonitorResultsQuery( + { + page: pageIndex, + size: pageSize, + search: searchQuery, + }, + { skip: isConfigLoading || !webMonitorEnabled }, + ); + + useEffect(() => { + if (isError && !!toast && webMonitorEnabled) { + toast({ + title: "Error fetching data", + description: "Please try again later", + status: "error", + }); + } + }, [isError, toast, webMonitorEnabled]); + + useEffect(() => { + if (data) { + setTotalPages(data.total || 1); + } + }, [data, setTotalPages]); + + const results = data?.items || []; + const loadingResults = isFetching + ? (Array.from({ length: pageSize }, (_, index) => ({ + key: index.toString(), + updates: [], + last_monitored: null, + })) as any[]) + : []; + + // TODO: [HJ-337] Add button functionality + + // const handleAdd = (monidorId: string) => { + // console.log("Add report", monidorId); + // }; + + const getWebsiteMonitorActions = useCallback( + (monitorKey: string) => [ + // , + + + , + ], + [], + ); + + if (!webMonitorEnabled) { + return ; + } + + return ( + + + + + + + + , + }} + renderItem={(summary: MonitorAggregatedResults) => { + return ( + !!summary && ( + + ) + ); + }} + /> + + {!!results && !!data?.total && data.total > pageSize && ( + <> + + + + )} + + ); +}; + +export default ActionCenterPage; diff --git a/clients/admin-ui/src/pages/taxonomy/index.tsx b/clients/admin-ui/src/pages/taxonomy/index.tsx index ee7a2ae02e..ac0c65d552 100644 --- a/clients/admin-ui/src/pages/taxonomy/index.tsx +++ b/clients/admin-ui/src/pages/taxonomy/index.tsx @@ -21,7 +21,10 @@ import PageHeader from "~/features/common/PageHeader"; import { errorToastParams, successToastParams } from "~/features/common/toast"; import TaxonomyEditDrawer from "~/features/taxonomy/components/TaxonomyEditDrawer"; import TaxonomyInteractiveTree from "~/features/taxonomy/components/TaxonomyInteractiveTree"; -import { CoreTaxonomiesEnum } from "~/features/taxonomy/constants"; +import { + CoreTaxonomiesEnum, + TAXONOMY_ROOT_NODE_ID, +} from "~/features/taxonomy/constants"; import useTaxonomySlices from "~/features/taxonomy/hooks/useTaxonomySlices"; import { TaxonomyEntity } from "~/features/taxonomy/types"; @@ -67,9 +70,11 @@ const TaxonomyPage: NextPage = () => { return; } + const isChildOfRoot = draftNewItem?.parent_key === TAXONOMY_ROOT_NODE_ID; const newItem = { ...draftNewItem, name: labelName, + parent_key: isChildOfRoot ? null : draftNewItem.parent_key, }; const result = await createTrigger(newItem); diff --git a/clients/admin-ui/src/theme/global.scss b/clients/admin-ui/src/theme/global.scss index 5c7aa053e1..d4357518ce 100644 --- a/clients/admin-ui/src/theme/global.scss +++ b/clients/admin-ui/src/theme/global.scss @@ -1,5 +1,19 @@ @import "fidesui/src/palette/palette.module.scss"; +/** + * Chakra removes heading font weight, wheras Ant assumes browser defaults. + * This sets the font weight for headings back to the browser default for Ant support. + * Remove this once Chakra has been removed. + */ +h1, +h2, +h3, +h4, +h5, +h6 { + font-weight: bold; +} + /** * Adds the color variables from the palette to the root element */ diff --git a/clients/admin-ui/src/types/common/PaginationQueryParams.ts b/clients/admin-ui/src/types/common/PaginationQueryParams.ts index bfdeda1475..0600daee7f 100644 --- a/clients/admin-ui/src/types/common/PaginationQueryParams.ts +++ b/clients/admin-ui/src/types/common/PaginationQueryParams.ts @@ -2,3 +2,11 @@ export interface PaginationQueryParams { page: number; size: number; } + +export interface PaginatedResponse { + items: T[]; + page: number; + size: number; + total: number; + pages: number; +} diff --git a/clients/fidesui/src/index.ts b/clients/fidesui/src/index.ts index 6179fa6da0..25652dce60 100644 --- a/clients/fidesui/src/index.ts +++ b/clients/fidesui/src/index.ts @@ -11,6 +11,7 @@ export type { FlexProps as AntFlexProps, FormInstance as AntFormInstance, InputProps as AntInputProps, + ListProps as AntListProps, SelectProps as AntSelectProps, SwitchProps as AntSwitchProps, GetProps, @@ -18,19 +19,23 @@ export type { } from "antd/lib"; export { Alert as AntAlert, + Avatar as AntAvatar, Breadcrumb as AntBreadcrumb, Button as AntButton, Card as AntCard, Checkbox as AntCheckbox, Col as AntCol, Divider as AntDivider, + Empty as AntEmpty, Flex as AntFlex, Form as AntForm, Input as AntInput, Layout as AntLayout, + List as AntList, Menu as AntMenu, Radio as AntRadio, Row as AntRow, + Skeleton as AntSkeleton, Space as AntSpace, Switch as AntSwitch, Tag as AntTag, @@ -41,6 +46,7 @@ export type { BreadcrumbItemType as AntBreadcrumbItemType, BreadcrumbProps as AntBreadcrumbProps, } from "antd/lib/breadcrumb/Breadcrumb"; +export type { ListItemProps as AntListItemProps } from "antd/lib/list"; export type { BaseOptionType as AntBaseOptionType, DefaultOptionType as AntDefaultOptionType, diff --git a/clients/package-lock.json b/clients/package-lock.json index 62d4f02527..b3a00b04f1 100644 --- a/clients/package-lock.json +++ b/clients/package-lock.json @@ -18,6 +18,7 @@ "admin-ui": { "dependencies": { "@ant-design/cssinjs": "^1.21.0", + "@date-fns/tz": "^1.2.0", "@fontsource/inter": "^4.5.15", "@monaco-editor/react": "^4.6.0", "@reduxjs/toolkit": "^1.9.3", @@ -28,8 +29,8 @@ "cytoscape": "^3.30.0", "cytoscape-klay": "^3.1.4", "d3-hierarchy": "^3.1.2", - "date-fns": "^2.29.3", - "date-fns-tz": "^2.0.0", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", "eslint-plugin-tailwindcss": "^3.17.4", "fides-js": "^0.0.1", "fidesui": "*", @@ -3058,6 +3059,12 @@ "ms": "^2.1.1" } }, + "node_modules/@date-fns/tz": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@date-fns/tz/-/tz-1.2.0.tgz", + "integrity": "sha512-LBrd7MiJZ9McsOgxqWX7AaxrDjcFVjWH/tIKJd7pnR7McaslGYOP1QmmiBXdJH/H/yLCT+rcQ7FaPBUxRGUtrg==", + "license": "MIT" + }, "node_modules/@emotion/babel-plugin": { "version": "11.11.0", "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz", @@ -10328,26 +10335,22 @@ } }, "node_modules/date-fns": { - "version": "2.30.0", - "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", - "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", - "dependencies": { - "@babel/runtime": "^7.21.0" - }, - "engines": { - "node": ">=0.11" - }, + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-4.1.0.tgz", + "integrity": "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==", + "license": "MIT", "funding": { - "type": "opencollective", - "url": "https://opencollective.com/date-fns" + "type": "github", + "url": "https://github.com/sponsors/kossnocorp" } }, "node_modules/date-fns-tz": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-2.0.1.tgz", - "integrity": "sha512-fJCG3Pwx8HUoLhkepdsP7Z5RsucUi+ZBOxyM5d0ZZ6c4SdYustq0VMmOu6Wf7bli+yS/Jwp91TOCqn9jMcVrUA==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-3.2.0.tgz", + "integrity": "sha512-sg8HqoTEulcbbbVXeg84u5UnlsQa8GS5QXMqjjYIhS4abEVVKIUwe0/l/UhrZdKaL/W5eWZNlbTeEIiOXTcsBQ==", + "license": "MIT", "peerDependencies": { - "date-fns": "2.x" + "date-fns": "^3.0.0 || ^4.0.0" } }, "node_modules/dayjs": { diff --git a/noxfiles/ci_nox.py b/noxfiles/ci_nox.py index d18428ee1d..8b835376a2 100644 --- a/noxfiles/ci_nox.py +++ b/noxfiles/ci_nox.py @@ -95,13 +95,19 @@ def xenon(session: nox.Session) -> None: "src", "tests", "scripts", - "--max-absolute B", - "--max-modules B", - "--max-average A", - "--ignore 'data, docs'", - "--exclude src/fides/_version.py", + "--max-absolute=B", + "--max-modules=B", + "--max-average=A", + "--ignore=data,docs", + "--exclude=src/fides/_version.py", + ) + session.run(*command, success_codes=[0, 1]) + session.warn( + "Note: This command was malformed so it's been failing to report complexity issues." + ) + session.warn( + "Intentionally suppressing the error status code for now to slowly work through the issues." ) - session.run(*command) ################## diff --git a/src/fides/api/api/v1/endpoints/dataset_endpoints.py b/src/fides/api/api/v1/endpoints/dataset_endpoints.py index 9240396295..cf443ed062 100644 --- a/src/fides/api/api/v1/endpoints/dataset_endpoints.py +++ b/src/fides/api/api/v1/endpoints/dataset_endpoints.py @@ -98,7 +98,7 @@ def _get_connection_config( connection_key: FidesKey, db: Session = Depends(deps.get_db) ) -> ConnectionConfig: - logger.info("Finding connection config with key '{}'", connection_key) + logger.debug("Finding connection config with key '{}'", connection_key) connection_config = ConnectionConfig.get_by(db, field="key", value=connection_key) if not connection_config: raise HTTPException( @@ -509,7 +509,7 @@ def get_datasets( Soon to be deprecated. """ - logger.info( + logger.debug( "Finding all datasets for connection '{}' with pagination params {}", connection_config.key, params, @@ -544,7 +544,7 @@ def get_dataset( Soon to be deprecated """ - logger.info( + logger.debug( "Finding dataset '{}' for connection '{}'", fides_key, connection_config.key ) dataset_config = DatasetConfig.filter( @@ -574,7 +574,7 @@ def get_dataset_configs( ) -> AbstractPage[DatasetConfig]: """Returns all Dataset Configs attached to current Connection Config.""" - logger.info( + logger.debug( "Finding all dataset configs for connection '{}' with pagination params {}", connection_config.key, params, @@ -598,7 +598,7 @@ def get_dataset_config( ) -> DatasetConfig: """Returns the specific Dataset Config linked to the Connection Config.""" - logger.info( + logger.debug( "Finding dataset config '{}' for connection '{}'", fides_key, connection_config.key, @@ -669,7 +669,7 @@ def get_ctl_datasets( Returns all CTL datasets . """ - logger.info( + logger.debug( f"Finding all datasets {remove_saas_datasets=} {only_unlinked_datasets=}" ) filters = [] diff --git a/src/fides/api/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/api/v1/endpoints/messaging_endpoints.py index 64ffc0097c..55c89a088a 100644 --- a/src/fides/api/api/v1/endpoints/messaging_endpoints.py +++ b/src/fides/api/api/v1/endpoints/messaging_endpoints.py @@ -182,7 +182,7 @@ def get_active_default_config(*, db: Session = Depends(deps.get_db)) -> Messagin """ Retrieves the active default messaging config. """ - logger.info("Finding active default messaging config") + logger.debug("Finding active default messaging config") try: messaging_config = MessagingConfig.get_active_default(db) except ValueError: @@ -421,7 +421,7 @@ def get_configs( """ Retrieves configs for messaging. """ - logger.info( + logger.debug( "Finding all messaging configurations with pagination params {}", params ) return paginate( @@ -441,7 +441,7 @@ def get_config_by_key( """ Retrieves configs for messaging service by key. """ - logger.info("Finding messaging config with key '{}'", config_key) + logger.debug("Finding messaging config with key '{}'", config_key) try: return get_messaging_config_by_key(db=db, key=config_key) @@ -463,7 +463,7 @@ def get_default_config_by_type( """ Retrieves default config for messaging service by type. """ - logger.info("Finding default messaging config of type '{}'", service_type) + logger.debug("Finding default messaging config of type '{}'", service_type) messaging_config = MessagingConfig.get_by_type(db, service_type) if not messaging_config: @@ -620,7 +620,7 @@ def get_default_messaging_template( """ Retrieves default messaging template by template type. """ - logger.info( + logger.debug( "Finding default messaging template of template type '{}'", template_type ) try: @@ -645,7 +645,7 @@ def get_messaging_template_by_id( """ Retrieves messaging template by template tid. """ - logger.info("Finding messaging template with id '{}'", template_id) + logger.debug("Finding messaging template with id '{}'", template_id) try: messaging_template = get_template_by_id(db, template_id) diff --git a/src/fides/api/api/v1/endpoints/policy_endpoints.py b/src/fides/api/api/v1/endpoints/policy_endpoints.py index 5e9a66483b..b0870e912b 100644 --- a/src/fides/api/api/v1/endpoints/policy_endpoints.py +++ b/src/fides/api/api/v1/endpoints/policy_endpoints.py @@ -56,14 +56,14 @@ def get_policy_list( """ Return a paginated list of all Policy records in this system """ - logger.info("Finding all policies with pagination params '{}'", params) + logger.debug("Finding all policies with pagination params '{}'", params) policies = Policy.query(db=db).order_by(Policy.created_at.desc()) return paginate(policies, params=params) def get_policy_or_error(db: Session, policy_key: FidesKey) -> Policy: """Helper method to load Policy or throw a 404""" - logger.info("Finding policy with key '{}'", policy_key) + logger.debug("Finding policy with key '{}'", policy_key) policy = Policy.get_by(db=db, field="key", value=policy_key) if not policy: raise HTTPException( @@ -160,7 +160,7 @@ def get_rule_or_error(db: Session, policy_key: FidesKey, rule_key: FidesKey) -> Also throws a 404 if a `Policy` with the given key can't be found. """ policy = get_policy_or_error(db, policy_key) - logger.info("Finding rule with key '{}'", rule_key) + logger.debug("Finding rule with key '{}'", rule_key) rule = Rule.filter( db=db, conditions=((Rule.policy_id == policy.id) & (Rule.key == rule_key)), @@ -191,7 +191,7 @@ def get_rule_list( Throws a 404 if the given `Policy` can't be found. """ policy = get_policy_or_error(db, policy_key) - logger.info( + logger.debug( "Finding all rules for policy {} with pagination params '{}'", policy_key, params, @@ -382,7 +382,7 @@ def get_rule_target_or_error( Helper method to load Rule Target or throw a 404. Also throws a 404 if a `Policy` or `Rule` with the given keys can't be found. """ - logger.info("Finding rule target with key '{}'", rule_target_key) + logger.debug("Finding rule target with key '{}'", rule_target_key) rule: Rule = get_rule_or_error(db, policy_key, rule_key) rule_target = RuleTarget.filter( db=db, @@ -417,7 +417,7 @@ def get_rule_target_list( Throws a 404 if the given `Rule` or `Policy` can't be found. """ rule = get_rule_or_error(db, policy_key, rule_key) - logger.info( + logger.debug( "Finding all rule targets for rule {} with pagination params '{}'", rule_key, params, diff --git a/src/fides/api/api/v1/endpoints/policy_webhook_endpoints.py b/src/fides/api/api/v1/endpoints/policy_webhook_endpoints.py index f1e5bc35f5..7b3c9715cc 100644 --- a/src/fides/api/api/v1/endpoints/policy_webhook_endpoints.py +++ b/src/fides/api/api/v1/endpoints/policy_webhook_endpoints.py @@ -50,7 +50,7 @@ def get_policy_pre_execution_webhooks( """ policy = get_policy_or_error(db, policy_key) - logger.info( + logger.debug( "Finding all Pre-Execution Webhooks for Policy '{}' with pagination params '{}'", policy.key, params, @@ -76,7 +76,7 @@ def get_policy_post_execution_webhooks( """ policy = get_policy_or_error(db, policy_key) - logger.info( + logger.debug( "Finding all Post-Execution Webhooks for Policy '{}' with pagination params '{}'", policy.key, params, @@ -218,7 +218,7 @@ def get_policy_webhook_or_error( Also verifies that the webhook belongs to the given Policy. """ - logger.info( + logger.debug( "Finding {}-Execution Webhook with key '{}' for Policy '{}'", webhook_cls.prefix.capitalize(), webhook_key, diff --git a/src/fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py b/src/fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py index 81d53b6a4d..88ab7f30b6 100644 --- a/src/fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py +++ b/src/fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py @@ -39,7 +39,9 @@ def get_pre_approval_webhook_list( """ Return a paginated list of all PreApprovalWebhook records in this system """ - logger.info("Finding all pre_approval webhooks with pagination params '{}'", params) + logger.debug( + "Finding all pre_approval webhooks with pagination params '{}'", params + ) pre_approval_webhooks = PreApprovalWebhook.query(db=db).order_by( PreApprovalWebhook.created_at.desc() ) @@ -50,7 +52,7 @@ def get_pre_approval_webhook_or_error( db: Session, webhook_key: FidesKey ) -> PreApprovalWebhook: """Helper method to load PreApprovalWebhook or throw a 404""" - logger.info("Finding PreApprovalWebhook with key '{}'", webhook_key) + logger.debug("Finding PreApprovalWebhook with key '{}'", webhook_key) pre_approval_webhook = PreApprovalWebhook.get_by( db=db, field="key", value=webhook_key ) diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index 3bec34ffed..bfe42a1455 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -203,7 +203,7 @@ def get_privacy_request_or_error( db: Session, privacy_request_id: str, error_if_deleted: Optional[bool] = True ) -> PrivacyRequest: """Load the privacy request or throw a 404""" - logger.info("Finding privacy request with id '{}'", privacy_request_id) + logger.debug("Finding privacy request with id '{}'", privacy_request_id) privacy_request = PrivacyRequest.get(db, object_id=privacy_request_id) @@ -708,7 +708,7 @@ def _shared_privacy_request_search( POST version of the endpoint. """ - logger.info("Finding all request statuses with pagination params {}", params) + logger.debug("Finding all request statuses with pagination params {}", params) query = db.query(PrivacyRequest) query = _filter_privacy_request_queryset( @@ -734,7 +734,7 @@ def _shared_privacy_request_search( include_deleted_requests, ) - logger.info( + logger.debug( "Sorting requests by field: {} and direction: {}", sort_field, sort_direction ) query = _sort_privacy_request_queryset(query, sort_field, sort_direction) @@ -921,7 +921,7 @@ def get_request_status_logs( get_privacy_request_or_error(db, privacy_request_id, error_if_deleted=False) - logger.info( + logger.debug( "Finding all execution logs for privacy request {} with params '{}'", privacy_request_id, params, diff --git a/src/fides/api/api/v1/endpoints/storage_endpoints.py b/src/fides/api/api/v1/endpoints/storage_endpoints.py index e4916e4e1c..58c82340b3 100644 --- a/src/fides/api/api/v1/endpoints/storage_endpoints.py +++ b/src/fides/api/api/v1/endpoints/storage_endpoints.py @@ -252,7 +252,7 @@ def get_configs( """ Retrieves configs for storage. """ - logger.info("Finding all storage configurations with pagination params {}", params) + logger.debug("Finding all storage configurations with pagination params {}", params) return paginate( StorageConfig.query(db).order_by(StorageConfig.created_at.desc()), params=params ) @@ -269,7 +269,7 @@ def get_config_by_key( """ Retrieves configs for storage by key. """ - logger.info("Finding storage config with key '{}'", config_key) + logger.debug("Finding storage config with key '{}'", config_key) storage_config = StorageConfig.get_by(db, field="key", value=config_key) if not storage_config: @@ -324,7 +324,7 @@ def get_active_default_config( """ Retrieves the active default storage config. """ - logger.info("Finding active default storage config") + logger.debug("Finding active default storage config") storage_config = get_active_default_storage_config(db) if not storage_config: raise HTTPException( @@ -568,7 +568,7 @@ def get_default_configs( """ Retrieves default configs for each storage types. """ - logger.info( + logger.debug( "Finding default storage configurations with pagination params {}", params ) return paginate( @@ -590,7 +590,7 @@ def get_default_config_by_type( """ Retrieves default config for given storage type. """ - logger.info("Finding default config for storage type '{}'", storage_type.value) + logger.debug("Finding default config for storage type '{}'", storage_type.value) storage_config = get_default_storage_config_by_type(db, storage_type) if not storage_config: raise HTTPException( diff --git a/src/fides/api/api/v1/endpoints/user_endpoints.py b/src/fides/api/api/v1/endpoints/user_endpoints.py index ab6b491a42..aeb9e674e4 100644 --- a/src/fides/api/api/v1/endpoints/user_endpoints.py +++ b/src/fides/api/api/v1/endpoints/user_endpoints.py @@ -509,7 +509,7 @@ def get_user(*, db: Session = Depends(get_db), user_id: str) -> FidesUser: if user is None: raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="User not found") - logger.info("Returning user with id: '{}'.", user_id) + logger.debug("Returning user with id: '{}'.", user_id) return user @@ -529,7 +529,7 @@ def get_users( if username: query = query.filter(FidesUser.username.ilike(f"%{escape_like(username)}%")) - logger.info("Returning a paginated list of users.") + logger.debug("Returning a paginated list of users.") return paginate(query.order_by(FidesUser.created_at.desc()), params=params) diff --git a/src/fides/api/models/db_cache.py b/src/fides/api/models/db_cache.py index 4102a0d942..7d19023d4b 100644 --- a/src/fides/api/models/db_cache.py +++ b/src/fides/api/models/db_cache.py @@ -87,3 +87,41 @@ def set_cache_value( db.commit() db.refresh(db_cache_entry) return db_cache_entry + + @classmethod + def delete_cache_entry( + cls, + db: Session, + namespace: DBCacheNamespace, + cache_key: str, + ) -> None: + """ + Deletes the cache entry for the given cache_key + """ + db.query(cls).filter( + cls.namespace == namespace.value, cls.cache_key == cache_key + ).delete() + db.commit() + + @classmethod + def clear_cache_for_namespace( + cls, + db: Session, + namespace: DBCacheNamespace, + ) -> None: + """ + Deletes all cache entries for the given namespace + """ + db.query(cls).filter(cls.namespace == namespace.value).delete() + db.commit() + + @classmethod + def clear_cache( + cls, + db: Session, + ) -> None: + """ + Deletes all cache entries + """ + db.query(cls).delete() + db.commit() diff --git a/tests/ops/models/test_dbcache.py b/tests/ops/models/test_dbcache.py index a1c48aec42..7fe9062fab 100644 --- a/tests/ops/models/test_dbcache.py +++ b/tests/ops/models/test_dbcache.py @@ -1,6 +1,14 @@ +from enum import Enum + from fides.api.models.db_cache import DBCache, DBCacheNamespace +# enum used to test extra namespaces since right now DBCacheNamespace only has one value +# this can be removed once more namespaces are added +class TestDbCacheNamespace(Enum): + TEST_NAMESPACE = "test-namespace" + + class TestDBCacheModel: def test_get_nonexisting_entry(self, db): cache_value = DBCache.get_cache_value( @@ -51,3 +59,179 @@ def test_update_cache_value(self, db): db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "some-key" ) assert updated_value.decode() == "value 2" + + def test_delete_cache_entry(self, db): + # Add two entries + DBCache.set_cache_value( + db, + DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, + "some-key", + "value 1".encode(), + ) + DBCache.set_cache_value( + db, + DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, + "some-key-2", + "value 2".encode(), + ) + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "some-key" + ).decode() + == "value 1" + ) + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "some-key-2" + ).decode() + == "value 2" + ) + + # Delete the first entry + DBCache.delete_cache_entry( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "some-key" + ) + + # Check the first entry was deleted + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "some-key" + ) + is None + ) + + # Check the second entry still exists + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "some-key-2" + ).decode() + == "value 2" + ) + + def test_clear_cache_for_namespace(self, db): + # Add three entries, two belonging to namespace LIST_PRIVACY_EXPERIENCE and one to another namespace + DBCache.set_cache_value( + db, + DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, + "key-1", + "value 1".encode(), + ) + DBCache.set_cache_value( + db, + DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, + "key-2", + "value 2".encode(), + ) + DBCache.set_cache_value( + db, + TestDbCacheNamespace.TEST_NAMESPACE, + "key-1", + "value 3".encode(), + ) + + # Check all entries exist + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "key-1" + ).decode() + == "value 1" + ) + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "key-2" + ).decode() + == "value 2" + ) + assert ( + DBCache.get_cache_value( + db, TestDbCacheNamespace.TEST_NAMESPACE, "key-1" + ).decode() + == "value 3" + ) + + # Clear the cache for LIST_PRIVACY_EXPERIENCE namespace + DBCache.clear_cache_for_namespace(db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE) + + # Check the entries belonging to LIST_PRIVACY_EXPERIENCE were deleted + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "key-1" + ) + is None + ) + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "key-2" + ) + is None + ) + + # Check the entry belonging to another namespace still exists + assert ( + DBCache.get_cache_value( + db, TestDbCacheNamespace.TEST_NAMESPACE, "key-1" + ).decode() + == "value 3" + ) + + def test_clear_cache(self, db): + # Add three entries, two belonging to namespace LIST_PRIVACY_EXPERIENCE and one to another namespace + DBCache.set_cache_value( + db, + DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, + "key-1", + "value 1".encode(), + ) + DBCache.set_cache_value( + db, + DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, + "key-2", + "value 2".encode(), + ) + DBCache.set_cache_value( + db, + TestDbCacheNamespace.TEST_NAMESPACE, + "key-1", + "value 3".encode(), + ) + + # Check all entries exist + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "key-1" + ).decode() + == "value 1" + ) + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "key-2" + ).decode() + == "value 2" + ) + assert ( + DBCache.get_cache_value( + db, TestDbCacheNamespace.TEST_NAMESPACE, "key-1" + ).decode() + == "value 3" + ) + + # Clear the cache + DBCache.clear_cache(db) + + # Check all entries were deleted + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "key-1" + ) + is None + ) + assert ( + DBCache.get_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "key-2" + ) + is None + ) + assert ( + DBCache.get_cache_value(db, TestDbCacheNamespace.TEST_NAMESPACE, "key-1") + is None + )