From 608b25c793469e2789c0dc7fe3466899828b8019 Mon Sep 17 00:00:00 2001 From: Dushyant Bhalgami Date: Thu, 11 Apr 2024 12:22:53 +0200 Subject: [PATCH 1/2] fix(ingestion/powerbi): patch column lineage for powerbi report --- .../api/incremental_lineage_helper.py | 62 ++++++++++++++++++- .../ingestion/source/powerbi/powerbi.py | 28 ++++++++- 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py index 397111705223c..d6e69dbdaa75a 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py @@ -1,4 +1,4 @@ -from typing import Iterable, Optional +from typing import Iterable, Optional, Union from pydantic.fields import Field @@ -6,11 +6,15 @@ from datahub.emitter.mce_builder import datahub_guid, set_aspect from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.schema_classes import ( + ChartInfoClass, + DashboardInfoClass, FineGrainedLineageClass, MetadataChangeEventClass, SystemMetadataClass, UpstreamLineageClass, ) +from datahub.specific.chart import ChartPatchBuilder +from datahub.specific.dashboard import DashboardPatchBuilder from datahub.specific.dataset import DatasetPatchBuilder @@ -28,6 +32,62 @@ def convert_upstream_lineage_to_patch( return MetadataWorkUnit(id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp) +def convert_chart_info_to_patch( + urn: str, aspect: ChartInfoClass, system_metadata: Optional[SystemMetadataClass] +) -> Union[MetadataWorkUnit, None]: + patch_builder = ChartPatchBuilder(urn, system_metadata) + + if aspect.customProperties: + for key in aspect.customProperties: + patch_builder.add_custom_property( + key, str(aspect.customProperties.get(key)) + ) + + if aspect.inputEdges: + for inputEdge in aspect.inputEdges: + patch_builder.add_input_edge(inputEdge) + + values = patch_builder.build() + + if values: + mcp = next(iter(values)) + return MetadataWorkUnit( + id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp + ) + else: + return None + + +def convert_dashboard_info_to_patch( + urn: str, aspect: DashboardInfoClass, system_metadata: Optional[SystemMetadataClass] +) -> Union[MetadataWorkUnit, None]: + patch_builder = DashboardPatchBuilder(urn, system_metadata) + + if aspect.customProperties: + for key in aspect.customProperties: + patch_builder.add_custom_property( + key, str(aspect.customProperties.get(key)) + ) + + if aspect.datasetEdges: + for datasetEdge in aspect.datasetEdges: + patch_builder.add_dataset_edge(datasetEdge) + + if aspect.chartEdges: + for chartEdge in aspect.chartEdges: + patch_builder.add_chart_edge(chartEdge) + + values = patch_builder.build() + + if values: + mcp = next(iter(values)) + return MetadataWorkUnit( + id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp + ) + else: + return None + + def get_fine_grained_lineage_key(fine_upstream: FineGrainedLineageClass) -> str: return datahub_guid( { diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py index 562fbf4f1a2c9..a9849bd504c1c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py @@ -84,6 +84,9 @@ from datahub.metadata.urns import ChartUrn from datahub.sql_parsing.sqlglot_lineage import ColumnLineageInfo from datahub.utilities.dedup_list import deduplicate_list +from src.datahub.ingestion.api.incremental_lineage_helper import ( + convert_dashboard_info_to_patch, +) # Logger instance logger = logging.getLogger(__name__) @@ -1300,17 +1303,36 @@ def get_workspace_workunit( # Convert PowerBi Dashboard and child entities to Datahub work unit to ingest into Datahub workunits = self.mapper.to_datahub_work_units(dashboard, workspace) for workunit in workunits: - # Return workunit to Datahub Ingestion framework - yield workunit + wu = self._get_dashboard_patch_work_unit(workunit) + if wu is not None: + yield wu for report in workspace.reports: for work_unit in self.mapper.report_to_datahub_work_units( report, workspace ): - yield work_unit + wu = self._get_dashboard_patch_work_unit(work_unit) + if wu is not None: + yield wu yield from self.extract_independent_datasets(workspace) + def _get_dashboard_patch_work_unit( + self, work_unit: MetadataWorkUnit + ) -> MetadataWorkUnit: + dashboard_info_aspect: Optional[ + DashboardInfoClass + ] = work_unit.get_aspect_of_type(DashboardInfoClass) + + if dashboard_info_aspect: + return convert_dashboard_info_to_patch( + work_unit.get_urn(), + dashboard_info_aspect, + work_unit.metadata.systemMetadata, + ) + else: + return work_unit + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: # As modified_workspaces is not idempotent, hence workunit processors are run later for each workspace_id # This will result in creating checkpoint for each workspace_id From cf57439bc82e52a26dcd782541363aa946698e20 Mon Sep 17 00:00:00 2001 From: Dushyant Bhalgami Date: Thu, 11 Apr 2024 13:25:19 +0200 Subject: [PATCH 2/2] fix(ingestion/powerbi): updated the golden files to fix the tests --- .../golden_test_admin_access_not_allowed.json | 40 +++---- .../powerbi/golden_test_admin_only.json | 71 +++-------- .../integration/powerbi/golden_test_cll.json | 40 +++---- .../powerbi/golden_test_container.json | 111 +++++------------- .../golden_test_disabled_ownership.json | 40 +++---- .../powerbi/golden_test_endorsement.json | 40 +++---- .../powerbi/golden_test_ingest.json | 40 +++---- .../powerbi/golden_test_lineage.json | 40 +++---- .../golden_test_lower_case_urn_ingest.json | 40 +++---- ..._config_and_modified_since_admin_only.json | 71 +++-------- .../golden_test_platform_instance_ingest.json | 40 +++---- .../powerbi/golden_test_report.json | 74 +++--------- .../golden_test_scan_all_workspaces.json | 77 +++++------- ...lden_test_server_to_platform_instance.json | 40 +++---- 14 files changed, 256 insertions(+), 508 deletions(-) diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json index c2f79ef6c1c2e..8ec431b6fe9f1 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json @@ -294,34 +294,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json index df66738f0fb01..4ca5cc4afd00d 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json @@ -1208,34 +1208,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1959,37 +1951,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", - "changeType": "UPSERT", - "aspectName": "dashboardInfo", - "aspect": { - "json": { - "customProperties": {}, - "title": "SalesMarketing", - "description": "Acryl sales marketing report", - "charts": [], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "dashboardUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 4d127e747d5eb..15f526406c438 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -1233,34 +1233,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json index 27fef20950c77..b65ba71c80159 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json @@ -1703,34 +1703,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2661,40 +2653,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", - "changeType": "UPSERT", - "aspectName": "dashboardInfo", - "aspect": { - "json": { - "customProperties": {}, - "title": "SalesMarketing", - "description": "Acryl sales marketing report", - "charts": [ - "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", - "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "dashboardUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", @@ -3089,31 +3047,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "0", - "workspaceName": "second-demo-workspace", - "workspaceId": "64ED5CAD-7C22-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "0" }, - "title": "test_dashboard2", - "description": "", - "charts": [], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "second-demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C22-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index 5a6d636018fdb..43a5aad3cd967 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -968,34 +968,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json index 8bdb8126a29a0..cf448776ba414 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json @@ -1140,34 +1140,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 707a3735900e4..566e405a03abb 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -1000,34 +1000,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json index 1a53d913bf38b..4f530acb11c80 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json @@ -1183,34 +1183,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json index 1132af6180d24..37ba034b08fe2 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json @@ -1000,34 +1000,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json index c89511da8b3e0..11a7fed6030ce 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json @@ -1010,34 +1010,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1151,37 +1143,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", - "changeType": "UPSERT", - "aspectName": "dashboardInfo", - "aspect": { - "json": { - "customProperties": {}, - "title": "SalesMarketing", - "description": "Acryl sales marketing report", - "charts": [], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "dashboardUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index 475a929edd6e1..12ce114482eee 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -1008,34 +1008,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json index 28cfe39b8e034..43092c4e20acb 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json @@ -1000,34 +1000,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1943,40 +1935,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", - "changeType": "UPSERT", - "aspectName": "dashboardInfo", - "aspect": { - "json": { - "customProperties": {}, - "title": "SalesMarketing", - "description": "Acryl sales marketing report", - "charts": [ - "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", - "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "dashboardUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json index 3be7402eed682..3c4499765e525 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json @@ -968,34 +968,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1077,31 +1069,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "0", - "workspaceName": "second-demo-workspace", - "workspaceId": "64ED5CAD-7C22-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "0" }, - "title": "test_dashboard2", - "description": "", - "charts": [], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "second-demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C22-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json index b6e377c539bd2..0c6239ae8d331 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json @@ -1208,34 +1208,26 @@ { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { - "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, - "dashboardUrl": "https://localhost/dashboards/web/1" - } + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + } + ] }, "systemMetadata": { "lastObserved": 1643871600000,