improve patch metadata

datahub-project · Jul 24, 2024 · f567258 · f567258
1 parent 1022d64
commit f567258
Show file tree

Hide file tree

Showing 20 changed files with 1,354 additions and 117 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
@@ -86,13 +86,13 @@ def convert_dashboard_info_to_patch(
         patch_builder.set_description(aspect.description)
 
     if aspect.charts:
-        patch_builder.set_charts(aspect.charts)
+        patch_builder.add_charts(aspect.charts)
 
     if aspect.dashboardUrl:
         patch_builder.set_dashboard_url(aspect.dashboardUrl)
 
     if aspect.datasets:
-        patch_builder.set_datasets(aspect.datasets)
+        patch_builder.add_datasets(aspect.datasets)
 
     if aspect.access:
         patch_builder.set_access(aspect.access)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
@@ -440,6 +440,11 @@ class PowerBiDashboardSourceConfig(
     )
     profiling: PowerBiProfilingConfig = PowerBiProfilingConfig()
 
+    patch_metadata: bool = pydantic.Field(
+        default=True,
+        description="Patch dashboard metadata",
+    )
+
     @root_validator(skip_on_failure=True)
     def validate_extract_column_level_lineage(cls, values: Dict) -> Dict:
         flags = [

diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
@@ -1382,7 +1382,7 @@ def _get_dashboard_patch_work_unit(
             DashboardInfoClass
         ] = work_unit.get_aspect_of_type(DashboardInfoClass)
 
-        if dashboard_info_aspect:
+        if dashboard_info_aspect and self.source_config.patch_metadata:
             return convert_dashboard_info_to_patch(
                 work_unit.get_urn(),
                 dashboard_info_aspect,

diff --git a/metadata-ingestion/src/datahub/specific/dashboard.py b/metadata-ingestion/src/datahub/specific/dashboard.py
@@ -450,25 +450,27 @@ def set_external_url(self, external_url: Optional[str]) -> "DashboardPatchBuilde
             )
         return self
 
-    def set_charts(self, charts: Optional[List[str]]) -> "DashboardPatchBuilder":
-        if charts:
-            self._add_patch(
-                DashboardInfo.ASPECT_NAME,
-                "add",
-                path="/charts",
-                value=charts,
-            )
+    def add_charts(self, chart_urns: Optional[List[str]]) -> "DashboardPatchBuilder":
+        if chart_urns:
+            for urn in chart_urns:
+                self._add_patch(
+                    aspect_name=DashboardInfo.ASPECT_NAME,
+                    op="add",
+                    path=f"/charts/{urn}",
+                    value=urn
+                )
 
         return self
 
-    def set_datasets(self, datasets: Optional[List[str]]) -> "DashboardPatchBuilder":
-        if datasets:
-            self._add_patch(
-                DashboardInfo.ASPECT_NAME,
-                "add",
-                path="/datasets",
-                value=datasets,
-            )
+    def add_datasets(self, dataset_urns: Optional[List[str]]) -> "DashboardPatchBuilder":
+        if dataset_urns:
+            for urn in dataset_urns:
+                self._add_patch(
+                    aspect_name=DashboardInfo.ASPECT_NAME,
+                    op="add",
+                    path=f"/datasets/{urn}",
+                    value=urn,
+                )
 
         return self
 

diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json
@@ -325,11 +325,13 @@
             },
             {
                 "op": "add",
-                "path": "/charts",
-                "value": [
-                    "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                    "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-                ]
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
             },
             {
                 "op": "add",

diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json
@@ -1234,11 +1234,13 @@
             },
             {
                 "op": "add",
-                "path": "/charts",
-                "value": [
-                    "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                    "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-                ]
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
             },
             {
                 "op": "add",

diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json
@@ -1289,11 +1289,13 @@
             },
             {
                 "op": "add",
-                "path": "/charts",
-                "value": [
-                    "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                    "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-                ]
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
             },
             {
                 "op": "add",

diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json
@@ -1734,11 +1734,13 @@
             },
             {
                 "op": "add",
-                "path": "/charts",
-                "value": [
-                    "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                    "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-                ]
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
             },
             {
                 "op": "add",
@@ -2949,21 +2951,28 @@
         "json": [
             {
                 "op": "add",
-                "path": "/charts",
-                "value": [
-                    "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)",
-                    "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)"
-                ]
+                "path": "/title",
+                "value": "SalesMarketing"
             },
             {
                 "op": "add",
-                "path": "/dashboardUrl",
-                "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715"
+                "path": "/description",
+                "value": "Acryl sales marketing report"
             },
             {
                 "op": "add",
-                "path": "/description",
-                "value": "Acryl sales marketing report"
+                "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)",
+                "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)",
+                "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)"
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715"
             },
             {
                 "op": "add",
@@ -2978,11 +2987,6 @@
                         "actor": "urn:li:corpuser:unknown"
                     }
                 }
-            },
-            {
-                "op": "add",
-                "path": "/title",
-                "value": "SalesMarketing"
             }
         ]
     },

diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json
@@ -999,11 +999,13 @@
             },
             {
                 "op": "add",
-                "path": "/charts",
-                "value": [
-                    "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                    "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-                ]
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
             },
             {
                 "op": "add",

diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json
@@ -1171,11 +1171,13 @@
             },
             {
                 "op": "add",
-                "path": "/charts",
-                "value": [
-                    "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                    "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-                ]
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
             },
             {
                 "op": "add",

diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json
@@ -1031,11 +1031,13 @@
             },
             {
                 "op": "add",
-                "path": "/charts",
-                "value": [
-                    "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
-                    "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
-                ]
+                "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)"
+            },
+            {
+                "op": "add",
+                "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)",
+                "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
             },
             {
                 "op": "add",