From ac2d43a8851cea04276ac41b4e72466e84a824ba Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Mon, 23 Sep 2024 21:01:58 +0530 Subject: [PATCH 1/4] fix(ingest/looker) : Handle DeserializeError to improve error reporting, underlying issue remains --- metadata-ingestion/setup.py | 2 ++ .../datahub/ingestion/source/looker/looker_common.py | 11 ++++++++++- .../datahub/ingestion/source/looker/looker_source.py | 11 +++++++++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index bf80172441405..41ba87b442b75 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -172,6 +172,8 @@ # LookML files with spaces between an item and the following comma. # See https://github.com/joshtemple/lkml/issues/73. "lkml>=1.3.4", + # Fixes https://github.com/looker-open-source/sdk-codegen/issues/1410 , cattrs appears to have introduced a breaking change starting in version 23.2. + "cattrs>=1.3,<23.2", *sqlglot_lib, "GitPython>2", "python-liquid", diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py index 1f54767de5a68..ff6b6dd907aad 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py @@ -21,6 +21,7 @@ ) from looker_sdk.error import SDKError +from looker_sdk.rtl.serialize import DeserializeError from looker_sdk.sdk.api40.models import ( LookmlModelExplore, LookmlModelExploreField, @@ -1131,7 +1132,15 @@ def from_api( # noqa: C901 logger.warning( f"Failed to extract explore {explore_name} from model {model}: {e}" ) - + except DeserializeError as e: + logger.warning( + f"Failed to extract explore {explore_name} from model {model}: {e}" + ) + reporter.report_warning( + title=f"Failed to extract explore {explore_name} from model {model}", + message="Encountered exception while attempting to find dependent views for this chart", + context=f"Explore: {explore_name}, Mode: {model}", + ) except AssertionError: reporter.report_warning( title="Unable to find Views", diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index e593e132dafd7..55e1ee8a5608e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -16,6 +16,7 @@ ) from looker_sdk.error import SDKError +from looker_sdk.rtl.serialize import DeserializeError from looker_sdk.sdk.api40.models import ( Dashboard, DashboardElement, @@ -1288,10 +1289,16 @@ def process_dashboard( dashboard_id=dashboard_id, fields=fields, ) - except SDKError: + except (SDKError, DeserializeError) as e: + logger.error(f"Failed to load dashboard from Looker API Error: {str(e)}") # A looker dashboard could be deleted in between the list and the get + error_type = ( + "Deserialization Error" + if isinstance(e, DeserializeError) + else "SDK Error" + ) self.reporter.report_warning( - title="Error Loading Dashboard", + title=f"Error Loading Dashboard: {error_type}", message="Error occurred while attempting to loading dashboard from Looker API. Skipping.", context=f"Dashboard ID: {dashboard_id}", ) From 6fb8c6606eb74b9bdf357d049f226d6cf183784e Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Wed, 25 Sep 2024 18:45:24 +0530 Subject: [PATCH 2/4] fix: Minor Change --- .../src/datahub/ingestion/source/looker/looker_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py index ff6b6dd907aad..ae4343092b4b4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py @@ -1134,7 +1134,7 @@ def from_api( # noqa: C901 ) except DeserializeError as e: logger.warning( - f"Failed to extract explore {explore_name} from model {model}: {e}" + f"Failed to extract explore {explore_name} from model {model}, Error: {e}" ) reporter.report_warning( title=f"Failed to extract explore {explore_name} from model {model}", From 4e8a707453826035a018a4cb1b723b756a5b6480 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Wed, 2 Oct 2024 14:08:34 +0530 Subject: [PATCH 3/4] fix: PR Comments --- metadata-ingestion/setup.py | 2 -- .../ingestion/source/looker/looker_common.py | 15 ++++++++------- .../ingestion/source/looker/looker_source.py | 8 ++------ 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 41ba87b442b75..bf80172441405 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -172,8 +172,6 @@ # LookML files with spaces between an item and the following comma. # See https://github.com/joshtemple/lkml/issues/73. "lkml>=1.3.4", - # Fixes https://github.com/looker-open-source/sdk-codegen/issues/1410 , cattrs appears to have introduced a breaking change starting in version 23.2. - "cattrs>=1.3,<23.2", *sqlglot_lib, "GitPython>2", "python-liquid", diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py index ae4343092b4b4..3cbb13375229b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py @@ -1133,13 +1133,14 @@ def from_api( # noqa: C901 f"Failed to extract explore {explore_name} from model {model}: {e}" ) except DeserializeError as e: - logger.warning( - f"Failed to extract explore {explore_name} from model {model}, Error: {e}" - ) - reporter.report_warning( - title=f"Failed to extract explore {explore_name} from model {model}", - message="Encountered exception while attempting to find dependent views for this chart", - context=f"Explore: {explore_name}, Mode: {model}", + reporter.warning( + title="Failed to fetch explore from the Looker API", + message=( + "An error occurred while extracting the explore from the model. " + "Please check the explore and model configurations." + ), + context=f"Explore: {explore_name}, Model: {model}", + exc=e, ) except AssertionError: reporter.report_warning( diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index 55e1ee8a5608e..099e510d05acb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -1292,15 +1292,11 @@ def process_dashboard( except (SDKError, DeserializeError) as e: logger.error(f"Failed to load dashboard from Looker API Error: {str(e)}") # A looker dashboard could be deleted in between the list and the get - error_type = ( - "Deserialization Error" - if isinstance(e, DeserializeError) - else "SDK Error" - ) self.reporter.report_warning( - title=f"Error Loading Dashboard: {error_type}", + title="Failed to fetch dashboard from the Looker API", message="Error occurred while attempting to loading dashboard from Looker API. Skipping.", context=f"Dashboard ID: {dashboard_id}", + exc=e, ) return [], None, dashboard_id, start_time, datetime.datetime.now() From 4d1360b3d1a8201a08e119282f38d9ebcce73c16 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 2 Oct 2024 11:50:40 -0700 Subject: [PATCH 4/4] Update metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py --- .../src/datahub/ingestion/source/looker/looker_source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index 099e510d05acb..f269ccf1cd98f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -1290,7 +1290,6 @@ def process_dashboard( fields=fields, ) except (SDKError, DeserializeError) as e: - logger.error(f"Failed to load dashboard from Looker API Error: {str(e)}") # A looker dashboard could be deleted in between the list and the get self.reporter.report_warning( title="Failed to fetch dashboard from the Looker API",