From 7ae901fc2b292c8c1345c97725d90435e245d7d6 Mon Sep 17 00:00:00 2001 From: Jason Gill Date: Thu, 30 Jan 2025 10:47:57 -0700 Subject: [PATCH 1/7] Fix bulk add/ignore assets (#5715) --- .../action-center/action-center.slice.ts | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts index dc0223fee4..3ace4109f5 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts @@ -83,23 +83,29 @@ const actionCenterApi = baseApi.injectEndpoints({ invalidatesTags: ["Discovery Monitor Results"], }), addMonitorResultAssets: build.mutation({ - query: (params) => ({ - method: "POST", - url: `/plus/discovery-monitor/promote`, - params: { - staged_resource_urns: params.urnList, - }, - }), + query: (params) => { + const queryParams = new URLSearchParams(); + params.urnList?.forEach((urn) => + queryParams.append("staged_resource_urns", urn), + ); + return { + method: "POST", + url: `/plus/discovery-monitor/promote?${queryParams}`, + }; + }, invalidatesTags: ["Discovery Monitor Results"], }), ignoreMonitorResultAssets: build.mutation({ - query: (params) => ({ - method: "POST", - url: `/plus/discovery-monitor/mute`, - params: { - staged_resource_urns: params.urnList, - }, - }), + query: (params) => { + const queryParams = new URLSearchParams(); + params.urnList?.forEach((urn) => + queryParams.append("staged_resource_urns", urn), + ); + return { + method: "POST", + url: `/plus/discovery-monitor/mute?${queryParams}`, + }; + }, invalidatesTags: ["Discovery Monitor Results"], }), updateAssetsSystem: build.mutation< From c0839fc270bba100459a86e24c766c249f3db940 Mon Sep 17 00:00:00 2001 From: Andres Torres Date: Thu, 30 Jan 2025 14:00:24 -0600 Subject: [PATCH 2/7] HA-418 - Added frequency field to DataHubSchema integration config (#5716) --- CHANGELOG.md | 3 ++- .../connection_secrets_datahub.py | 14 ++++++++++++++ .../endpoints/test_connection_config_endpoints.py | 6 +++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4b504011e..2ccb0f75ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,8 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ## [Unreleased](https://github.com/ethyca/fides/compare/2.54.0...main) - +## Changed +- Added frequency field to DataHubSchema integration config [#5716](https://github.com/ethyca/fides/pull/5716) ## [2.53.0](https://github.com/ethyca/fides/compare/2.53.0...2.54.0) diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py b/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py index bde79ad24f..bfda71ed50 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py @@ -1,3 +1,4 @@ +from enum import Enum from typing import ClassVar, List from pydantic import Field @@ -9,6 +10,14 @@ ) +class PeriodicIntegrationFrequency(Enum): + """Enum for periodic integration frequency""" + + daily = "daily" + weekly = "weekly" + monthly = "monthly" + + class DatahubSchema(ConnectionConfigSecretsSchema): datahub_server_url: AnyHttpUrlStringRemovesSlash = Field( title="DataHub Server URL", @@ -19,6 +28,11 @@ class DatahubSchema(ConnectionConfigSecretsSchema): description="The token used to authenticate with your DataHub server.", json_schema_extra={"sensitive": True}, ) + frequency: PeriodicIntegrationFrequency = Field( + title="Frequency", + description="The frequency at which the integration should run. Defaults to daily.", + default=PeriodicIntegrationFrequency.daily, + ) _required_components: ClassVar[List[str]] = ["datahub_server_url", "datahub_token"] diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 3bffd68796..83fade38c0 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -2018,6 +2018,7 @@ def test_put_datahub_connection_config_secrets( payload = { "datahub_server_url": "https://datahub.example.com", "datahub_token": "test", + "frequency": "weekly", } resp = api_client.put( url + "?verify=False", @@ -2034,6 +2035,7 @@ def test_put_datahub_connection_config_secrets( assert datahub_connection_config_no_secrets.secrets == { "datahub_server_url": "https://datahub.example.com", "datahub_token": "test", + "frequency": "weekly", } assert datahub_connection_config_no_secrets.last_test_timestamp is None assert datahub_connection_config_no_secrets.last_test_succeeded is None @@ -2069,6 +2071,7 @@ def test_put_datahub_connection_config_secrets_default_frequency( assert datahub_connection_config_no_secrets.secrets == { "datahub_server_url": "https://datahub.example.com", "datahub_token": "test", + "frequency": "daily", } assert datahub_connection_config_no_secrets.last_test_timestamp is None assert datahub_connection_config_no_secrets.last_test_succeeded is None @@ -2085,7 +2088,7 @@ def test_put_datahub_connection_config_secrets_missing_url( """ url = f"{V1_URL_PREFIX}{CONNECTIONS}/{datahub_connection_config_no_secrets.key}/secret" auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - payload = {"datahub_token": "test"} + payload = {"datahub_token": "test", "frequency": "weekly"} resp = api_client.put( url + "?verify=False", headers=auth_header, @@ -2111,6 +2114,7 @@ def test_put_datahub_connection_config_secrets_missing_token( auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) payload = { "datahub_server_url": "https://datahub.example.com", + "frequency": "weekly", } resp = api_client.put( url + "?verify=False", From c4d69ccb55793e152b0e29c7288dfd342176466f Mon Sep 17 00:00:00 2001 From: JadeWibbels Date: Fri, 31 Jan 2025 08:17:07 -0700 Subject: [PATCH 3/7] LJ-278 fix failing big query enterprise tests (#5713) Co-authored-by: Jade Wibbels --- CHANGELOG.md | 5 +- tests/fixtures/bigquery_fixtures.py | 45 ++- ...est_bigquery_enterprise_privacy_request.py | 331 +++++++----------- 3 files changed, 148 insertions(+), 233 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ccb0f75ec..d5fde119ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,9 +21,12 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ## [Unreleased](https://github.com/ethyca/fides/compare/2.54.0...main) -## Changed +### Changed - Added frequency field to DataHubSchema integration config [#5716](https://github.com/ethyca/fides/pull/5716) +### Fixed +- Fixed Bigquery flakey tests. [#5713](LJ-278-fix-failed-big-query-enterprise-tests) + ## [2.53.0](https://github.com/ethyca/fides/compare/2.53.0...2.54.0) ### Added diff --git a/tests/fixtures/bigquery_fixtures.py b/tests/fixtures/bigquery_fixtures.py index 982e18a12b..396e2525da 100644 --- a/tests/fixtures/bigquery_fixtures.py +++ b/tests/fixtures/bigquery_fixtures.py @@ -60,6 +60,17 @@ def bigquery_connection_config(db: Session, bigquery_keyfile_creds) -> Generator connection_config.delete(db) +@pytest.fixture(scope="function") +def bigquery_enterprise_test_dataset_collections( + example_datasets: List[Dict], +) -> List[str]: + """Returns the names of collections in the BigQuery Enterprise dataset""" + bigquery_enterprise_dataset = example_datasets[16] + return [ + collection["name"] for collection in bigquery_enterprise_dataset["collections"] + ] + + @pytest.fixture(scope="function") def bigquery_enterprise_connection_config( db: Session, bigquery_enterprise_keyfile_creds @@ -341,6 +352,8 @@ def bigquery_example_test_dataset_config_with_namespace_and_partitioning_meta( def bigquery_resources( bigquery_example_test_dataset_config, ): + # Increment the ids by a random number to avoid conflicts on concurrent test runs + random_increment = random.randint(1, 99999) bigquery_connection_config = bigquery_example_test_dataset_config.connection_config connector = BigQueryConnector(bigquery_connection_config) bigquery_client = connector.client() @@ -351,11 +364,11 @@ def bigquery_resources( stmt = "select max(id) from customer;" res = connection.execute(stmt) - customer_id = res.all()[0][0] + 1 + customer_id = res.all()[0][0] + random_increment stmt = "select max(id) from address;" res = connection.execute(stmt) - address_id = res.all()[0][0] + 1 + address_id = res.all()[0][0] + random_increment city = "Test City" state = "TX" @@ -382,7 +395,7 @@ def bigquery_resources( stmt = "select max(id) from employee;" res = connection.execute(stmt) - employee_id = res.all()[0][0] + 1 + employee_id = res.all()[0][0] + random_increment employee_email = f"employee-{uuid}@example.com" employee_name = f"Jane {uuid}" @@ -422,6 +435,8 @@ def bigquery_resources( def bigquery_resources_with_namespace_meta( bigquery_example_test_dataset_config_with_namespace_meta, ): + # Increment the ids by a random number to avoid conflicts on concurrent test runs + random_increment = random.randint(1, 99999) bigquery_connection_config = ( bigquery_example_test_dataset_config_with_namespace_meta.connection_config ) @@ -434,11 +449,11 @@ def bigquery_resources_with_namespace_meta( stmt = "select max(id) from fidesopstest.customer;" res = connection.execute(stmt) - customer_id = res.all()[0][0] + 1 + customer_id = res.all()[0][0] + random_increment stmt = "select max(id) from fidesopstest.address;" res = connection.execute(stmt) - address_id = res.all()[0][0] + 1 + address_id = res.all()[0][0] + random_increment city = "Test City" state = "TX" @@ -465,7 +480,7 @@ def bigquery_resources_with_namespace_meta( stmt = "select max(id) from fidesopstest.employee;" res = connection.execute(stmt) - employee_id = res.all()[0][0] + 1 + employee_id = res.all()[0][0] + random_increment employee_email = f"employee-{uuid}@example.com" employee_name = f"Jane {uuid}" @@ -505,6 +520,8 @@ def bigquery_resources_with_namespace_meta( def bigquery_enterprise_resources( bigquery_enterprise_test_dataset_config, ): + # Increment the ids by a random number to avoid conflicts on concurrent test runs + random_increment = random.randint(1, 99999) bigquery_connection_config = ( bigquery_enterprise_test_dataset_config.connection_config ) @@ -515,8 +532,6 @@ def bigquery_enterprise_resources( # Real max id in the Stackoverflow dataset is 20081052, so we purposefully generate and id above this max stmt = "select max(id) from enterprise_dsr_testing.users;" res = connection.execute(stmt) - # Increment the id by a random number to avoid conflicts on concurrent test runs - random_increment = random.randint(0, 99999) user_id = res.all()[0][0] + random_increment display_name = ( f"fides_testing_{user_id}" # prefix to do manual cleanup if needed @@ -536,7 +551,6 @@ def bigquery_enterprise_resources( post_body = "For me, the solution was to adopt 3 cats and dance with them under the full moon at midnight." stmt = "select max(id) from enterprise_dsr_testing.stackoverflow_posts_partitioned;" res = connection.execute(stmt) - random_increment = random.randint(0, 99999) post_id = res.all()[0][0] + random_increment stmt = f""" insert into enterprise_dsr_testing.stackoverflow_posts_partitioned (body, creation_date, id, owner_user_id, owner_display_name) @@ -547,7 +561,6 @@ def bigquery_enterprise_resources( # Create test comments data. Comments are responses to posts or questions on Stackoverflow, and does not include original question or post itself. stmt = "select max(id) from enterprise_dsr_testing.comments;" res = connection.execute(stmt) - random_increment = random.randint(0, 99999) comment_id = res.all()[0][0] + random_increment comment_text = "FYI this only works if you have pytest installed locally." stmt = f""" @@ -557,9 +570,8 @@ def bigquery_enterprise_resources( connection.execute(stmt) # Create test post_history data - stmt = "select max(id) from enterprise_dsr_testing.comments;" + stmt = "select max(id) from enterprise_dsr_testing.post_history;" res = connection.execute(stmt) - random_increment = random.randint(0, 99999) post_history_id = res.all()[0][0] + random_increment revision_text = "this works if you have pytest" uuid = str(uuid4()) @@ -600,6 +612,8 @@ def bigquery_enterprise_resources( def bigquery_enterprise_resources_with_partitioning( bigquery_enterprise_test_dataset_config_with_partitioning_meta, ): + # Increment the ids by a random number to avoid conflicts on concurrent test runs + random_increment = random.randint(1, 99999) bigquery_connection_config = ( bigquery_enterprise_test_dataset_config_with_partitioning_meta.connection_config ) @@ -610,8 +624,6 @@ def bigquery_enterprise_resources_with_partitioning( # Real max id in the Stackoverflow dataset is 20081052, so we purposefully generate and id above this max stmt = "select max(id) from enterprise_dsr_testing.users;" res = connection.execute(stmt) - # Increment the id by a random number to avoid conflicts on concurrent test runs - random_increment = random.randint(0, 99999) user_id = res.all()[0][0] + random_increment display_name = ( f"fides_testing_{user_id}" # prefix to do manual cleanup if needed @@ -631,7 +643,6 @@ def bigquery_enterprise_resources_with_partitioning( post_body = "For me, the solution was to adopt 3 cats and dance with them under the full moon at midnight." stmt = "select max(id) from enterprise_dsr_testing.stackoverflow_posts_partitioned;" res = connection.execute(stmt) - random_increment = random.randint(0, 99999) post_id = res.all()[0][0] + random_increment stmt = f""" insert into enterprise_dsr_testing.stackoverflow_posts_partitioned (body, creation_date, id, owner_user_id, owner_display_name) @@ -642,7 +653,6 @@ def bigquery_enterprise_resources_with_partitioning( # Create test comments data. Comments are responses to posts or questions on Stackoverflow, and does not include original question or post itself. stmt = "select max(id) from enterprise_dsr_testing.comments;" res = connection.execute(stmt) - random_increment = random.randint(0, 99999) comment_id = res.all()[0][0] + random_increment comment_text = "FYI this only works if you have pytest installed locally." stmt = f""" @@ -652,9 +662,8 @@ def bigquery_enterprise_resources_with_partitioning( connection.execute(stmt) # Create test post_history data - stmt = "select max(id) from enterprise_dsr_testing.comments;" + stmt = "select max(id) from enterprise_dsr_testing.post_history;" res = connection.execute(stmt) - random_increment = random.randint(0, 99999) post_history_id = res.all()[0][0] + random_increment revision_text = "this works if you have pytest" uuid = str(uuid4()) diff --git a/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py b/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py index 9042d4758a..d62f9fbd05 100644 --- a/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py +++ b/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py @@ -1,9 +1,11 @@ +from typing import Any, Dict, List, Optional from unittest import mock import pytest from fides.api.models.audit_log import AuditLog, AuditLogAction -from fides.api.models.privacy_request import ExecutionLog +from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest +from fides.api.util.collection_util import Row from tests.ops.service.privacy_request.test_request_runner_service import ( get_privacy_request_results, ) @@ -13,6 +15,107 @@ PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL = 150 +def validate_privacy_request( + pr: PrivacyRequest, + user_id: int, + bigquery_enterprise_test_dataset_collections: List[str], + access: bool = True, +) -> Dict[str, Optional[List[Row]]]: + """ + Validates the results of a privacy request with assertions. + - Checks that all collections have been queried + - Checks that all keys have a non-empty value + - Checks that only results for the user_id are returned + - Checks that the expected number of records are returned for each collection + + Note: The access boolean determines if we are looking at the access or erasure result counts. + + """ + results = pr.get_raw_access_results() + + assert len(results.keys()) == len(bigquery_enterprise_test_dataset_collections) + + for key in results.keys(): + assert results[key] is not None + assert results[key] != {} + + users = results["enterprise_dsr_testing:users"] + assert len(users) == 1 + user_details = users[0] + assert user_details["id"] == user_id + + assert ( + len( + [ + comment["user_id"] + for comment in results["enterprise_dsr_testing:comments"] + ] + ) + == 16 + if access + else 1 + ) + assert ( + len( + [post["user_id"] for post in results["enterprise_dsr_testing:post_history"]] + ) + == 39 + if access + else 1 + ) + assert ( + len( + [ + post["title"] + for post in results[ + "enterprise_dsr_testing:stackoverflow_posts_partitioned" + ] + ] + ) + == 30 + if access + else 1 + ) + + return results + + +def validate_erasure_privacy_request( + bigquery_enterprise_resources: dict[str, Any], user_id: int +) -> None: + """Validates the results of an erasure request with assertions.""" + bigquery_client = bigquery_enterprise_resources["client"] + post_history_id = bigquery_enterprise_resources["post_history_id"] + comment_id = bigquery_enterprise_resources["comment_id"] + post_id = bigquery_enterprise_resources["post_id"] + with bigquery_client.connect() as connection: + stmt = f"select text from enterprise_dsr_testing.post_history where id = {post_history_id};" + res = connection.execute(stmt).all() + for row in res: + assert row.text is None + + stmt = f"select user_display_name, text from enterprise_dsr_testing.comments where id = {comment_id};" + res = connection.execute(stmt).all() + for row in res: + assert row.user_display_name is None + assert row.text is None + + stmt = f"select owner_user_id, owner_display_name, body from enterprise_dsr_testing.stackoverflow_posts_partitioned where id = {post_id};" + res = connection.execute(stmt).all() + for row in res: + assert ( + row.owner_user_id == bigquery_enterprise_resources["user_id"] + ) # not targeted by policy + assert row.owner_display_name is None + assert row.body is None + + stmt = f"select display_name, location from enterprise_dsr_testing.users where id = {user_id};" + res = connection.execute(stmt).all() + for row in res: + assert row.display_name is None + assert row.location is None + + @pytest.mark.integration_bigquery @pytest.mark.integration_external @pytest.mark.parametrize( @@ -37,6 +140,7 @@ def test_access_request( policy_pre_execution_webhooks, policy_post_execution_webhooks, run_privacy_request_task, + bigquery_enterprise_test_dataset_collections, ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 request.getfixturevalue( @@ -67,44 +171,7 @@ def test_access_request( PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL, ) - results = pr.get_raw_access_results() - assert len(results.keys()) == 4 - - for key in results.keys(): - assert results[key] is not None - assert results[key] != {} - - users = results["enterprise_dsr_testing:users"] - assert len(users) == 1 - user_details = users[0] - assert user_details["id"] == user_id - - assert ( - len( - [ - comment["user_id"] - for comment in results["enterprise_dsr_testing:comments"] - ] - ) - == 16 - ) - assert ( - len( - [post["user_id"] for post in results["enterprise_dsr_testing:post_history"]] - ) - == 39 - ) - assert ( - len( - [ - post["title"] - for post in results[ - "enterprise_dsr_testing:stackoverflow_posts_partitioned" - ] - ] - ) - == 30 - ) + validate_privacy_request(pr, user_id, bigquery_enterprise_test_dataset_collections) log_id = pr.execution_logs[0].id pr_id = pr.id @@ -156,10 +223,10 @@ def test_erasure_request( bigquery_fixtures, bigquery_enterprise_erasure_policy, run_privacy_request_task, + bigquery_enterprise_test_dataset_collections, ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 bigquery_enterprise_resources = request.getfixturevalue(bigquery_fixtures) - bigquery_client = bigquery_enterprise_resources["client"] # first test access request against manually added data user_id = bigquery_enterprise_resources["user_id"] @@ -184,43 +251,8 @@ def test_erasure_request( PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL, ) - results = pr.get_raw_access_results() - assert len(results.keys()) == 4 - - for key in results.keys(): - assert results[key] is not None - assert results[key] != {} - - users = results["enterprise_dsr_testing:users"] - assert len(users) == 1 - user_details = users[0] - assert user_details["id"] == user_id - - assert ( - len( - [ - comment["user_id"] - for comment in results["enterprise_dsr_testing:comments"] - ] - ) - == 1 - ) - assert ( - len( - [post["user_id"] for post in results["enterprise_dsr_testing:post_history"]] - ) - == 1 - ) - assert ( - len( - [ - post["title"] - for post in results[ - "enterprise_dsr_testing:stackoverflow_posts_partitioned" - ] - ] - ) - == 1 + validate_privacy_request( + pr, user_id, bigquery_enterprise_test_dataset_collections, False ) data = { @@ -230,7 +262,7 @@ def test_erasure_request( "email": customer_email, "stackoverflow_user_id": { "label": "Stackoverflow User Id", - "value": bigquery_enterprise_resources["user_id"], + "value": user_id, }, }, } @@ -245,36 +277,7 @@ def test_erasure_request( ) pr.delete(db=db) - bigquery_client = bigquery_enterprise_resources["client"] - post_history_id = bigquery_enterprise_resources["post_history_id"] - comment_id = bigquery_enterprise_resources["comment_id"] - post_id = bigquery_enterprise_resources["post_id"] - with bigquery_client.connect() as connection: - stmt = f"select text from enterprise_dsr_testing.post_history where id = {post_history_id};" - res = connection.execute(stmt).all() - for row in res: - assert row.text is None - - stmt = f"select user_display_name, text from enterprise_dsr_testing.comments where id = {comment_id};" - res = connection.execute(stmt).all() - for row in res: - assert row.user_display_name is None - assert row.text is None - - stmt = f"select owner_user_id, owner_display_name, body from enterprise_dsr_testing.stackoverflow_posts_partitioned where id = {post_id};" - res = connection.execute(stmt).all() - for row in res: - assert ( - row.owner_user_id == bigquery_enterprise_resources["user_id"] - ) # not targeted by policy - assert row.owner_display_name is None - assert row.body is None - - stmt = f"select display_name, location from enterprise_dsr_testing.users where id = {user_id};" - res = connection.execute(stmt).all() - for row in res: - assert row.display_name is None - assert row.location is None + validate_erasure_privacy_request(bigquery_enterprise_resources, user_id) @pytest.mark.integration_bigquery @@ -295,6 +298,7 @@ def test_access_request_multiple_custom_identities( policy_pre_execution_webhooks, policy_post_execution_webhooks, run_privacy_request_task, + bigquery_enterprise_test_dataset_collections, ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 @@ -321,44 +325,7 @@ def test_access_request_multiple_custom_identities( PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL, ) - results = pr.get_raw_access_results() - assert len(results.keys()) == 4 - - for key in results.keys(): - assert results[key] is not None - assert results[key] != {} - - users = results["enterprise_dsr_testing:users"] - assert len(users) == 1 - user_details = users[0] - assert user_details["id"] == user_id - - assert ( - len( - [ - comment["user_id"] - for comment in results["enterprise_dsr_testing:comments"] - ] - ) - == 16 - ) - assert ( - len( - [post["user_id"] for post in results["enterprise_dsr_testing:post_history"]] - ) - == 39 - ) - assert ( - len( - [ - post["title"] - for post in results[ - "enterprise_dsr_testing:stackoverflow_posts_partitioned" - ] - ] - ) - == 30 - ) + validate_privacy_request(pr, user_id, bigquery_enterprise_test_dataset_collections) log_id = pr.execution_logs[0].id pr_id = pr.id @@ -410,10 +377,10 @@ def test_erasure_request_multiple_custom_identities( bigquery_fixtures, bigquery_enterprise_erasure_policy, run_privacy_request_task, + bigquery_enterprise_test_dataset_collections, ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 bigquery_enterprise_resources = request.getfixturevalue(bigquery_fixtures) - bigquery_client = bigquery_enterprise_resources["client"] # first test access request against manually added data user_id = bigquery_enterprise_resources["user_id"] @@ -437,43 +404,8 @@ def test_erasure_request_multiple_custom_identities( PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL, ) - results = pr.get_raw_access_results() - assert len(results.keys()) == 4 - - for key in results.keys(): - assert results[key] is not None - assert results[key] != {} - - users = results["enterprise_dsr_testing:users"] - assert len(users) == 1 - user_details = users[0] - assert user_details["id"] == user_id - - assert ( - len( - [ - comment["user_id"] - for comment in results["enterprise_dsr_testing:comments"] - ] - ) - == 1 - ) - assert ( - len( - [post["user_id"] for post in results["enterprise_dsr_testing:post_history"]] - ) - == 1 - ) - assert ( - len( - [ - post["title"] - for post in results[ - "enterprise_dsr_testing:stackoverflow_posts_partitioned" - ] - ] - ) - == 1 + validate_privacy_request( + pr, user_id, bigquery_enterprise_test_dataset_collections, False ) data = { @@ -497,33 +429,4 @@ def test_erasure_request_multiple_custom_identities( ) pr.delete(db=db) - bigquery_client = bigquery_enterprise_resources["client"] - post_history_id = bigquery_enterprise_resources["post_history_id"] - comment_id = bigquery_enterprise_resources["comment_id"] - post_id = bigquery_enterprise_resources["post_id"] - with bigquery_client.connect() as connection: - stmt = f"select text from enterprise_dsr_testing.post_history where id = {post_history_id};" - res = connection.execute(stmt).all() - for row in res: - assert row.text is None - - stmt = f"select user_display_name, text from enterprise_dsr_testing.comments where id = {comment_id};" - res = connection.execute(stmt).all() - for row in res: - assert row.user_display_name is None - assert row.text is None - - stmt = f"select owner_user_id, owner_display_name, body from enterprise_dsr_testing.stackoverflow_posts_partitioned where id = {post_id};" - res = connection.execute(stmt).all() - for row in res: - assert ( - row.owner_user_id == bigquery_enterprise_resources["user_id"] - ) # not targeted by policy - assert row.owner_display_name is None - assert row.body is None - - stmt = f"select display_name, location from enterprise_dsr_testing.users where id = {user_id};" - res = connection.execute(stmt).all() - for row in res: - assert row.display_name is None - assert row.location is None + validate_erasure_privacy_request(bigquery_enterprise_resources, user_id) From 50f44b8b383d7d16cd95b460b337cb8782cd6af1 Mon Sep 17 00:00:00 2001 From: Jason Gill Date: Mon, 3 Feb 2025 11:32:11 -0700 Subject: [PATCH 4/7] Convert getWebsiteIconUrl util to use Brandfetch api (#5712) --- clients/admin-ui/cypress/e2e/action-center.cy.ts | 2 ++ .../admin-ui/src/features/common/util.test.ts | 16 ++++++++++++++++ clients/admin-ui/src/features/common/utils.ts | 4 ++-- .../action-center/MonitorResult.tsx | 14 ++++++++++++-- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/clients/admin-ui/cypress/e2e/action-center.cy.ts b/clients/admin-ui/cypress/e2e/action-center.cy.ts index 628b7728da..fbff87b231 100644 --- a/clients/admin-ui/cypress/e2e/action-center.cy.ts +++ b/clients/admin-ui/cypress/e2e/action-center.cy.ts @@ -65,6 +65,8 @@ describe("Action center", () => { const monitorKey = result .attr("data-testid") .replace("monitor-result-", ""); + // property icon + cy.wrap(result).find(".ant-list-item-meta-avatar").should("exist"); // linked title cy.wrap(result) .contains("assets detected") diff --git a/clients/admin-ui/src/features/common/util.test.ts b/clients/admin-ui/src/features/common/util.test.ts index d1aa050a6e..7c43be1ae1 100644 --- a/clients/admin-ui/src/features/common/util.test.ts +++ b/clients/admin-ui/src/features/common/util.test.ts @@ -5,6 +5,7 @@ import { getKeysFromMap, getOptionsFromMap, getPII, + getWebsiteIconUrl, } from "~/features/common/utils"; // TODO: add tests for the other utils functions @@ -84,3 +85,18 @@ describe(getOptionsFromMap.name, () => { expect(result).toEqual([]); }); }); + +describe(getWebsiteIconUrl.name, () => { + it("should return the icon URL", () => { + const result = getWebsiteIconUrl("example.com"); + expect(result).toEqual( + "https://cdn.brandfetch.io/example.com/icon/theme/light/fallback/404/h/24/w/24?c=1idbRjELpikqQ1PLiqb", + ); + }); + it("should return the icon URL with a custom size", () => { + const result = getWebsiteIconUrl("example.com", 56); + expect(result).toEqual( + "https://cdn.brandfetch.io/example.com/icon/theme/light/fallback/404/h/56/w/56?c=1idbRjELpikqQ1PLiqb", + ); + }); +}); diff --git a/clients/admin-ui/src/features/common/utils.ts b/clients/admin-ui/src/features/common/utils.ts index ab18bc6803..31c41d75b7 100644 --- a/clients/admin-ui/src/features/common/utils.ts +++ b/clients/admin-ui/src/features/common/utils.ts @@ -117,6 +117,6 @@ export const getOptionsFromMap = ( value: key, })); -export const getWebsiteIconUrl = (hostname: string) => { - return `https://icons.duckduckgo.com/ip3/${hostname}.ico`; +export const getWebsiteIconUrl = (domain: string, size = 24) => { + return `https://cdn.brandfetch.io/${domain}/icon/theme/light/fallback/404/h/${size}/w/${size}?c=1idbRjELpikqQ1PLiqb`; }; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx index 3614217ffe..e3d0c07d24 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx @@ -63,13 +63,23 @@ export const MonitorResult = ({ } + avatar={ + } + style={{ + backgroundColor: "transparent", + color: "var(--ant-color-text)", + }} + /> + } title={ - {`${totalUpdates} assets detected${property ? `on ${property}` : ""}`} + {`${totalUpdates} assets detected${property ? ` on ${property}` : ""}`} {!!warning && ( Date: Mon, 3 Feb 2025 14:40:37 -0600 Subject: [PATCH 5/7] Fix data catalog breadcrumb navigation (#5717) --- CHANGELOG.md | 1 + .../admin-ui/cypress/e2e/data-catalog.cy.ts | 2 +- .../CatalogResourcesTable.tsx | 18 +---- .../systems/CatalogSystemsTable.tsx | 6 +- .../data-catalog/utils/urnParsing.test.ts | 63 +++++++++++++++ .../data-catalog/utils/urnParsing.tsx | 80 +++++++++++++++++++ .../projects/[projectUrn]/[resourceUrn].tsx | 56 +++++++++++++ .../index.tsx} | 14 ++-- .../{ => resources}/[resourceUrn].tsx | 22 +++-- .../[systemId]/{ => resources}/index.tsx | 2 +- .../admin-ui/src/pages/data-catalog/index.tsx | 4 +- 11 files changed, 236 insertions(+), 32 deletions(-) create mode 100644 clients/admin-ui/src/features/data-catalog/utils/urnParsing.test.ts create mode 100644 clients/admin-ui/src/features/data-catalog/utils/urnParsing.tsx create mode 100644 clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].tsx rename clients/admin-ui/src/pages/data-catalog/[systemId]/projects/{[projectId].tsx => [projectUrn]/index.tsx} (88%) rename clients/admin-ui/src/pages/data-catalog/[systemId]/{ => resources}/[resourceUrn].tsx (68%) rename clients/admin-ui/src/pages/data-catalog/[systemId]/{ => resources}/index.tsx (97%) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5fde119ea..ad596dad53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Fixed - Fixed Bigquery flakey tests. [#5713](LJ-278-fix-failed-big-query-enterprise-tests) +- Fixed breadcrumb navigation issues in data catalog view [#5717](https://github.com/ethyca/fides/pull/5717) ## [2.53.0](https://github.com/ethyca/fides/compare/2.53.0...2.54.0) diff --git a/clients/admin-ui/cypress/e2e/data-catalog.cy.ts b/clients/admin-ui/cypress/e2e/data-catalog.cy.ts index 2eda2079b4..25e9799953 100644 --- a/clients/admin-ui/cypress/e2e/data-catalog.cy.ts +++ b/clients/admin-ui/cypress/e2e/data-catalog.cy.ts @@ -97,7 +97,7 @@ describe("data catalog", () => { beforeEach(() => { stubStagedResourceActions(); cy.visit( - `${DATA_CATALOG_ROUTE}/bigquery_system/monitor.project.test_dataset_1`, + `${DATA_CATALOG_ROUTE}/bigquery_system/projects/monitor.project/monitor.project.test_dataset_1`, ); }); diff --git a/clients/admin-ui/src/features/data-catalog/staged-resources/CatalogResourcesTable.tsx b/clients/admin-ui/src/features/data-catalog/staged-resources/CatalogResourcesTable.tsx index bcb8b9600e..97d8183beb 100644 --- a/clients/admin-ui/src/features/data-catalog/staged-resources/CatalogResourcesTable.tsx +++ b/clients/admin-ui/src/features/data-catalog/staged-resources/CatalogResourcesTable.tsx @@ -6,10 +6,8 @@ import { useReactTable, } from "@tanstack/react-table"; import { Box, Flex } from "fidesui"; -import { useRouter } from "next/router"; import { useEffect, useMemo, useState } from "react"; -import { DATA_CATALOG_ROUTE } from "~/features/common/nav/v2/routes"; import { FidesTableV2, PaginationBar, @@ -24,11 +22,7 @@ import { SearchInput } from "~/features/data-discovery-and-detection/SearchInput import { StagedResourceType } from "~/features/data-discovery-and-detection/types/StagedResourceType"; import { findResourceType } from "~/features/data-discovery-and-detection/utils/findResourceType"; import resourceHasChildren from "~/features/data-discovery-and-detection/utils/resourceHasChildren"; -import { - DiffStatus, - StagedResourceAPIResponse, - SystemResponse, -} from "~/types/api"; +import { DiffStatus, StagedResourceAPIResponse } from "~/types/api"; // everything except muted const DIFF_STATUS_FILTERS = [ @@ -53,13 +47,11 @@ const EMPTY_RESPONSE = { const CatalogResourcesTable = ({ resourceUrn, - system, + onRowClick, }: { resourceUrn: string; - system: SystemResponse; + onRowClick: (row: StagedResourceAPIResponse) => void; }) => { - const router = useRouter(); - const { PAGE_SIZES, pageSize, @@ -135,9 +127,7 @@ const CatalogResourcesTable = ({ tableInstance={tableInstance} emptyTableNotice={} getRowIsClickable={(row) => resourceHasChildren(row)} - onRowClick={(row) => - router.push(`${DATA_CATALOG_ROUTE}/${system.fides_key}/${row.urn}`) - } + onRowClick={onRowClick} /> (); -const SystemsTable = () => { +const CatalogSystemsTable = () => { const [rowSelectionState, setRowSelectionState] = useState( {}, ); @@ -92,7 +92,7 @@ const SystemsTable = () => { "monitor_config_ids", ); - const url = `${DATA_CATALOG_ROUTE}/${row.fides_key}${hasProjects ? "/projects" : ""}?${queryString}`; + const url = `${DATA_CATALOG_ROUTE}/${row.fides_key}/${hasProjects ? "projects" : "resources"}?${queryString}`; router.push(url); }; @@ -183,4 +183,4 @@ const SystemsTable = () => { ); }; -export default SystemsTable; +export default CatalogSystemsTable; diff --git a/clients/admin-ui/src/features/data-catalog/utils/urnParsing.test.ts b/clients/admin-ui/src/features/data-catalog/utils/urnParsing.test.ts new file mode 100644 index 0000000000..d3ca498b3e --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/utils/urnParsing.test.ts @@ -0,0 +1,63 @@ +import { + parseResourceBreadcrumbsNoProject, + parseResourceBreadcrumbsWithProject, +} from "~/features/data-catalog/utils/urnParsing"; + +const URL_PREFIX = "/url-prefix"; + +describe(parseResourceBreadcrumbsWithProject.name, () => { + const URN = "monitor.project.dataset.table.field"; + const EXPECTED_TITLES = ["project", "dataset", "table", "field"]; + const EXPECTED_HREFS = [ + "/url-prefix/monitor.project", + "/url-prefix/monitor.project/monitor.project.dataset", + "/url-prefix/monitor.project/monitor.project.dataset.table", + undefined, + ]; + + it("should return no breadcrumbs without a URN", () => { + const result = parseResourceBreadcrumbsWithProject(undefined, URL_PREFIX); + expect(result).toEqual([]); + }); + + it("should return no breadcrumbs with a short URN", () => { + const result = parseResourceBreadcrumbsWithProject("monitor", URL_PREFIX); + expect(result).toEqual([]); + }); + + it("should return the correct breadcrumbs when URN is provided", () => { + const result = parseResourceBreadcrumbsWithProject(URN, URL_PREFIX); + result.forEach((breadcrumb, idx) => { + expect(breadcrumb.title).toEqual(EXPECTED_TITLES[idx]); + expect(breadcrumb.href).toEqual(EXPECTED_HREFS[idx]); + }); + }); +}); + +describe(parseResourceBreadcrumbsNoProject.name, () => { + const URN = "monitor.dataset.table.field"; + const EXPECTED_TITLES = ["dataset", "table", "field"]; + const EXPECTED_HREFS = [ + "/url-prefix/monitor.dataset", + "/url-prefix/monitor.dataset.table", + undefined, + ]; + + it("should return no breadcrumbs without a URN", () => { + const result = parseResourceBreadcrumbsNoProject(undefined, URL_PREFIX); + expect(result).toEqual([]); + }); + + it("should return no breadcrumbs with a short URN", () => { + const result = parseResourceBreadcrumbsNoProject("monitor", URL_PREFIX); + expect(result).toEqual([]); + }); + + it("should return the correct breadcrumbs when URN is provided", () => { + const result = parseResourceBreadcrumbsNoProject(URN, URL_PREFIX); + result.forEach((breadcrumb, idx) => { + expect(breadcrumb.title).toEqual(EXPECTED_TITLES[idx]); + expect(breadcrumb.href).toEqual(EXPECTED_HREFS[idx]); + }); + }); +}); diff --git a/clients/admin-ui/src/features/data-catalog/utils/urnParsing.tsx b/clients/admin-ui/src/features/data-catalog/utils/urnParsing.tsx new file mode 100644 index 0000000000..4f8eb5a33e --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/utils/urnParsing.tsx @@ -0,0 +1,80 @@ +import { Icons } from "fidesui"; + +import { NextBreadcrumbProps } from "~/features/common/nav/v2/NextBreadcrumb"; + +const URN_SEPARATOR = "."; + +export const getProjectName = (urn: string) => { + const urnParts = urn.split(URN_SEPARATOR); + return urnParts[1]; +}; + +const RESOURCE_ICONS = [ + , + , + , +]; + +export const parseResourceBreadcrumbsWithProject = ( + urn: string | undefined, + urlPrefix: string, +) => { + if (!urn) { + return []; + } + const urnParts = urn.split(URN_SEPARATOR); + if (urnParts.length < 2) { + return []; + } + const projectUrn = urnParts.splice(0, 2).join(URN_SEPARATOR); + const subResourceUrns: NextBreadcrumbProps["items"] = []; + + urnParts.reduce((prev, current, idx) => { + const isLast = idx === urnParts.length - 1; + const next = `${prev}${URN_SEPARATOR}${current}`; + subResourceUrns.push({ + title: current, + href: !isLast ? `${urlPrefix}/${projectUrn}/${next}` : undefined, + icon: RESOURCE_ICONS[idx], + }); + return next; + }, projectUrn); + + return [ + { + title: getProjectName(projectUrn), + href: `${urlPrefix}/${projectUrn}`, + icon: , + }, + ...subResourceUrns, + ]; +}; + +export const parseResourceBreadcrumbsNoProject = ( + urn: string | undefined, + urlPrefix: string, +) => { + if (!urn) { + return []; + } + + const urnParts = urn.split(URN_SEPARATOR); + if (urnParts.length < 2) { + return []; + } + const monitorId = urnParts.shift(); + const subResourceUrns: NextBreadcrumbProps["items"] = []; + + urnParts.reduce((prev, current, idx) => { + const isLast = idx === urnParts.length - 1; + const next = `${prev}${URN_SEPARATOR}${current}`; + subResourceUrns.push({ + title: current, + href: !isLast ? `${urlPrefix}/${next}` : undefined, + icon: RESOURCE_ICONS[idx], + }); + return next; + }, monitorId); + + return subResourceUrns; +}; diff --git a/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].tsx b/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].tsx new file mode 100644 index 0000000000..35528fce86 --- /dev/null +++ b/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].tsx @@ -0,0 +1,56 @@ +import { NextPage } from "next"; +import { useRouter } from "next/router"; + +import FidesSpinner from "~/features/common/FidesSpinner"; +import Layout from "~/features/common/Layout"; +import { DATA_CATALOG_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import CatalogResourcesTable from "~/features/data-catalog/staged-resources/CatalogResourcesTable"; +import { parseResourceBreadcrumbsWithProject } from "~/features/data-catalog/utils/urnParsing"; +import { useGetSystemByFidesKeyQuery } from "~/features/system"; + +const CatalogResourceView: NextPage = () => { + const { query } = useRouter(); + const systemId = query.systemId as string; + const projectUrn = query.projectUrn as string; + const resourceUrn = query.resourceUrn as string; + const { data: system, isLoading } = useGetSystemByFidesKeyQuery(systemId); + + const router = useRouter(); + + const resourceBreadcrumbs = + parseResourceBreadcrumbsWithProject( + resourceUrn, + `${DATA_CATALOG_ROUTE}/${systemId}/projects`, + ) ?? []; + + if (isLoading) { + return ; + } + + return ( + + + + router.push( + `${DATA_CATALOG_ROUTE}/${system!.fides_key}/projects/${projectUrn}/${row.urn}`, + ) + } + /> + + ); +}; + +export default CatalogResourceView; diff --git a/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectId].tsx b/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectUrn]/index.tsx similarity index 88% rename from clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectId].tsx rename to clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectUrn]/index.tsx index 43d75f053a..32f7b95ce2 100644 --- a/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectId].tsx +++ b/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectUrn]/index.tsx @@ -4,6 +4,7 @@ import { getGroupedRowModel, useReactTable, } from "@tanstack/react-table"; +import { Icons } from "fidesui"; import { useRouter } from "next/router"; import { useEffect, useMemo } from "react"; @@ -18,6 +19,7 @@ import { } from "~/features/common/table/v2"; import EmptyCatalogTableNotice from "~/features/data-catalog/datasets/EmptyCatalogTableNotice"; import useCatalogDatasetColumns from "~/features/data-catalog/datasets/useCatalogDatasetColumns"; +import { getProjectName } from "~/features/data-catalog/utils/urnParsing"; import { useGetMonitorResultsQuery } from "~/features/data-discovery-and-detection/discovery-detection.slice"; import { useGetSystemByFidesKeyQuery } from "~/features/system"; import { StagedResourceAPIResponse } from "~/types/api"; @@ -33,7 +35,7 @@ const EMPTY_RESPONSE = { const CatalogDatasetView = () => { const { query, push } = useRouter(); const systemKey = query.systemId as string; - const projectId = query.projectId as string; + const projectUrn = query.projectUrn as string; const { data: system, isLoading: systemIsLoading } = useGetSystemByFidesKeyQuery(systemKey); @@ -57,7 +59,7 @@ const CatalogDatasetView = () => { isLoading, data: resources, } = useGetMonitorResultsQuery({ - staged_resource_urn: projectId, + staged_resource_urn: projectUrn, page: pageIndex, size: pageSize, }); @@ -94,9 +96,9 @@ const CatalogDatasetView = () => { { title: "All systems", href: DATA_CATALOG_ROUTE }, { title: system?.name || systemKey, - href: `${DATA_CATALOG_ROUTE}/${systemKey}/projects`, + href: DATA_CATALOG_ROUTE, }, - { title: projectId }, + { title: getProjectName(projectUrn), icon: }, ]} /> {!showContent && } @@ -106,7 +108,9 @@ const CatalogDatasetView = () => { tableInstance={tableInstance} emptyTableNotice={} onRowClick={(row) => - push(`${DATA_CATALOG_ROUTE}/${systemKey}/${row.urn}`) + push( + `${DATA_CATALOG_ROUTE}/${systemKey}/projects/${projectUrn}/${row.urn}/`, + ) } /> { @@ -15,9 +15,12 @@ const CatalogResourceView: NextPage = () => { const resourceUrn = query.resourceUrn as string; const { data: system, isLoading } = useGetSystemByFidesKeyQuery(systemId); - const resourceBreadcrumbs = - parseUrnToBreadcrumbs(resourceUrn, `${DATA_CATALOG_ROUTE}/${systemId}`) ?? - []; + const router = useRouter(); + + const resourceBreadcrumbs = parseResourceBreadcrumbsNoProject( + resourceUrn, + `${DATA_CATALOG_ROUTE}/${systemId}/resources`, + ); if (isLoading) { return ; @@ -31,12 +34,19 @@ const CatalogResourceView: NextPage = () => { { title: "All systems", href: DATA_CATALOG_ROUTE }, { title: system?.name ?? system?.fides_key, - href: `${DATA_CATALOG_ROUTE}/${systemId}`, + href: `${DATA_CATALOG_ROUTE}`, }, ...resourceBreadcrumbs, ]} /> - + + router.push( + `${DATA_CATALOG_ROUTE}/${system!.fides_key}/resources/${row.urn}`, + ) + } + /> ); }; diff --git a/clients/admin-ui/src/pages/data-catalog/[systemId]/index.tsx b/clients/admin-ui/src/pages/data-catalog/[systemId]/resources/index.tsx similarity index 97% rename from clients/admin-ui/src/pages/data-catalog/[systemId]/index.tsx rename to clients/admin-ui/src/pages/data-catalog/[systemId]/resources/index.tsx index 4f8163317c..54cb63015a 100644 --- a/clients/admin-ui/src/pages/data-catalog/[systemId]/index.tsx +++ b/clients/admin-ui/src/pages/data-catalog/[systemId]/resources/index.tsx @@ -104,7 +104,7 @@ const CatalogDatasetViewNoProjects = () => { tableInstance={tableInstance} emptyTableNotice={} onRowClick={(row) => - push(`${DATA_CATALOG_ROUTE}/${systemKey}/${row.urn}`) + push(`${DATA_CATALOG_ROUTE}/${systemKey}/resources/${row.urn}`) } /> { return ( @@ -9,7 +9,7 @@ const DataCatalogMainPage = () => { heading="Data catalog" breadcrumbItems={[{ title: "All systems" }]} /> - + ); }; From da19f8b328e7c2857d9b1ca210d97f841e78c24e Mon Sep 17 00:00:00 2001 From: JadeWibbels Date: Mon, 3 Feb 2025 14:59:27 -0700 Subject: [PATCH 6/7] fixed link for changelog (#5728) Co-authored-by: Jade Wibbels --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad596dad53..ccf905b7e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Added frequency field to DataHubSchema integration config [#5716](https://github.com/ethyca/fides/pull/5716) ### Fixed -- Fixed Bigquery flakey tests. [#5713](LJ-278-fix-failed-big-query-enterprise-tests) +- Fixed Bigquery flakey tests. [#5713](https://github.com/ethyca/fides/pull/5713) - Fixed breadcrumb navigation issues in data catalog view [#5717](https://github.com/ethyca/fides/pull/5717) ## [2.53.0](https://github.com/ethyca/fides/compare/2.53.0...2.54.0) From df9aecdc02f64de18642529cd81213bb7d5167bd Mon Sep 17 00:00:00 2001 From: Andres Torres Date: Mon, 3 Feb 2025 17:41:18 -0600 Subject: [PATCH 7/7] HA-368 - Fix: fides annotate dataset enters incorrect value (#5727) --- CHANGELOG.md | 1 + noxfiles/git_nox.py | 1 + src/fides/core/annotate_dataset.py | 9 +- tests/ctl/cli/test_cli.py | 22 +++ tests/ctl/data/failing_direction.yml | 249 +++++++++++++++++++++++++++ 5 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 tests/ctl/data/failing_direction.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf905b7e8..9ddf689db4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Added frequency field to DataHubSchema integration config [#5716](https://github.com/ethyca/fides/pull/5716) ### Fixed +- Fixed `fides annotate dataset` command enters incorrect value on the `direction` field. [#5727](https://github.com/ethyca/fides/pull/5727) - Fixed Bigquery flakey tests. [#5713](https://github.com/ethyca/fides/pull/5713) - Fixed breadcrumb navigation issues in data catalog view [#5717](https://github.com/ethyca/fides/pull/5717) diff --git a/noxfiles/git_nox.py b/noxfiles/git_nox.py index 5cec609db8..c56b7cd1b8 100644 --- a/noxfiles/git_nox.py +++ b/noxfiles/git_nox.py @@ -85,6 +85,7 @@ def tag(session: nox.Session, action: str) -> None: - tag(dry) = Show the tag that would be applied. - tag(push) = Tag the current commit and push it. NOTE: This will trigger a new CI job to publish the tag. """ + # pip3.10 install GitPython from git.repo import Repo repo = Repo() diff --git a/src/fides/core/annotate_dataset.py b/src/fides/core/annotate_dataset.py index 9954b9169d..e44999d35a 100644 --- a/src/fides/core/annotate_dataset.py +++ b/src/fides/core/annotate_dataset.py @@ -158,12 +158,16 @@ def annotate_dataset( if include_null: output_dataset.append(current_dataset.model_dump(mode="json")) else: - output_dataset.append(current_dataset.model_dump(exclude_none=True)) + output_dataset.append( + current_dataset.model_dump(mode="json", exclude_none=True) + ) except AnnotationAbortError: if include_null: output_dataset.append(current_dataset.model_dump(mode="json")) else: - output_dataset.append(current_dataset.model_dump(exclude_none=True)) + output_dataset.append( + current_dataset.model_dump(mode="json", exclude_none=True) + ) break manifests.write_manifest(dataset_file, output_dataset, "dataset") echo_green("Annotation process complete.") @@ -200,7 +204,6 @@ def annotate_fields( """ Check for data_categories at the field level """ - for field in get_all_level_fields(table.fields): if not field.data_categories: click.secho( diff --git a/tests/ctl/cli/test_cli.py b/tests/ctl/cli/test_cli.py index 1d2e4938bb..2161efd052 100644 --- a/tests/ctl/cli/test_cli.py +++ b/tests/ctl/cli/test_cli.py @@ -325,6 +325,28 @@ def test_annotate( assert result.exit_code == 0 print(result.output) + def test_regression_annotate_dataset( + self, + test_config_path: str, + test_cli_runner: CliRunner, + ): + test_cli_runner.invoke( + cli, + [ + "-f", + test_config_path, + "annotate", + "dataset", + "tests/ctl/data/failing_direction.yml", + ], + input="user\n", + ) + with open("tests/ctl/data/failing_direction.yml", "r") as dataset_yml: + try: + dataset_yml = yaml.safe_load(dataset_yml) + except yaml.constructor.ConstructorError: + assert False, "The yaml file is not valid" + @pytest.mark.integration def test_audit(test_config_path: str, test_cli_runner: CliRunner) -> None: diff --git a/tests/ctl/data/failing_direction.yml b/tests/ctl/data/failing_direction.yml new file mode 100644 index 0000000000..8f23699039 --- /dev/null +++ b/tests/ctl/data/failing_direction.yml @@ -0,0 +1,249 @@ +dataset: +- fides_key: google_cloud_sql_postgres_example_test_dataset + organization_fides_key: default_organization + name: Google Cloud SQL for Postgres Example Test Dataset + description: Example of a Google Cloud SQL Postgres dataset containing a variety + of related tables like customers, products, addresses, etc. + collections: + - name: address + fields: + - name: city + data_categories: + - user.contact.address.city + - name: house + data_categories: + - user.contact.address.street + - name: id + data_categories: + - system.operations + fides_meta: + primary_key: true + - name: state + data_categories: + - user.contact.address.state + - name: street + data_categories: + - user.contact.address.street + - name: zip + data_categories: + - user.contact.address.postal_code + - name: customer + fields: + - name: address_id + data_categories: + - system.operations + fides_meta: + references: + - dataset: google_cloud_sql_postgres_example_test_dataset + field: address.id + direction: to + - name: created + data_categories: + - system.operations + - name: email + data_categories: + - user.contact.email + fides_meta: + identity: email + data_type: string + - name: id + data_categories: + - user.unique_id + fides_meta: + primary_key: true + - name: name + data_categories: + - user.name + - name: employee + fields: + - name: address_id + data_categories: + - system.operations + fides_meta: + references: + - dataset: google_cloud_sql_postgres_example_test_dataset + field: address.id + direction: to + - name: email + data_categories: + - user.contact.email + fides_meta: + identity: email + data_type: string + - name: id + data_categories: + - user.unique_id + fides_meta: + primary_key: true + - name: name + data_categories: + - user.name + - name: login + fields: + - name: customer_id + data_categories: + - user.unique_id + fides_meta: + references: + - dataset: google_cloud_sql_postgres_example_test_dataset + field: customer.id + direction: from + - name: id + data_categories: + - system.operations + - name: time + data_categories: + - user.sensor + - name: order_item + fields: + - name: order_id + data_categories: + - system.operations + fides_meta: + references: + - dataset: google_cloud_sql_postgres_example_test_dataset + field: orders.id + direction: from + - name: product_id + data_categories: + - system.operations + fides_meta: + references: + - dataset: google_cloud_sql_postgres_example_test_dataset + field: product.id + direction: to + - name: quantity + data_categories: + - system.operations + - name: orders + fields: + - name: customer_id + data_categories: + - user.unique_id + fides_meta: + references: + - dataset: google_cloud_sql_postgres_example_test_dataset + field: customer.id + direction: from + - name: id + data_categories: + - system.operations + fides_meta: + primary_key: true + - name: shipping_address_id + data_categories: + - system.operations + fides_meta: + references: + - dataset: google_cloud_sql_postgres_example_test_dataset + field: address.id + direction: to + - name: payment_card + fields: + - name: billing_address_id + data_categories: + - system.operations + fides_meta: + references: + - dataset: google_cloud_sql_postgres_example_test_dataset + field: address.id + direction: to + - name: ccn + data_categories: + - user.financial.bank_account + - name: code + data_categories: + - user.financial + - name: customer_id + data_categories: + - user.unique_id + fides_meta: + references: + - dataset: google_cloud_sql_postgres_example_test_dataset + field: customer.id + direction: from + - name: id + data_categories: + - system.operations + - name: name + data_categories: + - user.financial + - name: preferred + data_categories: + - user + - name: product + fields: + - name: id + data_categories: + - system.operations + - name: name + data_categories: + - system.operations + - name: price + data_categories: + - system.operations + - name: report + fields: + - name: email + data_categories: + - user.contact.email + fides_meta: + identity: email + data_type: string + - name: id + data_categories: + - system.operations + - name: month + data_categories: + - system.operations + - name: name + data_categories: + - system.operations + - name: total_visits + data_categories: + - system.operations + - name: year + data_categories: + - system.operations + - name: service_request + fields: + - name: alt_email + data_categories: + - user.contact.email + fides_meta: + identity: email + data_type: string + - name: closed + data_categories: + - system.operations + - name: email + data_categories: + - system.operations + fides_meta: + identity: email + data_type: string + - name: employee_id + data_categories: + - user.unique_id + fides_meta: + references: + - dataset: google_cloud_sql_postgres_example_test_dataset + field: employee.id + direction: from + - name: id + data_categories: + - system.operations + - name: opened + data_categories: + - system.operations + - name: visit + fields: + - name: email + data_categories: + - user.contact.email + fides_meta: + identity: email + data_type: string + - name: last_visit + data_categories: + - system.operations