fix(ingest/looker) : Handle DeserializeError to improve error reporti…

…ng, underlying issue remains
datahub-project · Sep 24, 2024 · 9d3adbf · 9d3adbf
1 parent 315ff8f
commit 9d3adbf
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 3 deletions.
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
@@ -172,6 +172,8 @@
     # LookML files with spaces between an item and the following comma.
     # See https://github.com/joshtemple/lkml/issues/73.
     "lkml>=1.3.4",
+    # Fixes https://github.com/looker-open-source/sdk-codegen/issues/1410 , cattrs appears to have introduced a breaking change starting in version 23.2.
+    "cattrs>=1.3,<23.2",
     *sqlglot_lib,
     "GitPython>2",
     "python-liquid",

diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
@@ -21,6 +21,7 @@
 )
 
 from looker_sdk.error import SDKError
+from looker_sdk.rtl.serialize import DeserializeError
 from looker_sdk.sdk.api40.models import (
     LookmlModelExplore,
     LookmlModelExploreField,
@@ -1131,7 +1132,15 @@ def from_api(  # noqa: C901
                 logger.warning(
                     f"Failed to extract explore {explore_name} from model {model}: {e}"
                 )
-
+        except DeserializeError as e:
+            logger.warning(
+                f"Failed to extract explore {explore_name} from model {model}: {e}"
+            )
+            reporter.report_warning(
+                title=f"Failed to extract explore {explore_name} from model {model}",
+                message="Encountered exception while attempting to find dependent views for this chart",
+                context=f"Explore: {explore_name}, Mode: {model}",
+            )
         except AssertionError:
             reporter.report_warning(
                 title="Unable to find Views",

diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
@@ -17,6 +17,7 @@
 )
 
 from looker_sdk.error import SDKError
+from looker_sdk.rtl.serialize import DeserializeError
 from looker_sdk.sdk.api40.models import (
     Dashboard,
     DashboardElement,
@@ -1309,10 +1310,16 @@ def process_dashboard(
                 dashboard_id=dashboard_id,
                 fields=fields,
             )
-        except SDKError:
+        except (SDKError, DeserializeError) as e:
+            logger.error(f"Failed to load dashboard from Looker API Error: {str(e)}")
             # A looker dashboard could be deleted in between the list and the get
+            error_type = (
+                "Deserialization Error"
+                if isinstance(e, DeserializeError)
+                else "SDK Error"
+            )
             self.reporter.report_warning(
-                title="Error Loading Dashboard",
+                title=f"Error Loading Dashboard: {error_type}",
                 message="Error occurred while attempting to loading dashboard from Looker API. Skipping.",
                 context=f"Dashboard ID: {dashboard_id}",
             )