fix(ingestion/sac): handle descriptions which are None correctly

datahub-project · Oct 9, 2024 · edde82a · edde82a
1 parent ef4805d
commit edde82a
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 10 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py
@@ -329,7 +329,7 @@ def get_resource_workunits(
             entityUrn=dashboard_urn,
             aspect=DashboardInfoClass(
                 title=resource.name,
-                description=resource.description,
+                description=resource.description if resource.description is not None else "",
                 lastModified=ChangeAuditStampsClass(
                     created=AuditStampClass(
                         time=round(resource.created_time.timestamp() * 1000),
@@ -559,7 +559,7 @@ def get_sac_connection(
 
         retries = 3
         backoff_factor = 10
-        status_forcelist = (500,)
+        status_forcelist = (400, 500, 503)
 
         retry = Retry(
             total=retries,
@@ -611,7 +611,7 @@ def get_resources(self) -> Iterable[Resource]:
         entity: pyodata.v2.service.EntityProxy
         for entity in entities:
             resource_id: str = entity.resourceId
-            name: str = entity.name.strip()
+            name: str = entity.name.strip() if entity.name is not None else entity.resourceId
 
             if not self.config.resource_id_pattern.allowed(
                 resource_id
@@ -655,8 +655,8 @@ def get_resources(self) -> Iterable[Resource]:
                     ResourceModel(
                         namespace=namespace,
                         model_id=model_id,
-                        name=nav_entity.name.strip(),
-                        description=nav_entity.description.strip(),
+                        name=nav_entity.name.strip() if nav_entity.name is not None else f"{namespace}:{model_id}",
+                        description=nav_entity.description.strip() if nav_entity.description is not None else None,
                         system_type=nav_entity.systemType,  # BW or HANA
                         connection_id=nav_entity.connectionId,
                         external_id=nav_entity.externalId,  # query:[][][query] or view:[schema][schema.namespace][view]
@@ -678,7 +678,7 @@ def get_resources(self) -> Iterable[Resource]:
                 resource_subtype=entity.resourceSubtype,
                 story_id=entity.storyId,
                 name=name,
-                description=entity.description.strip(),
+                description=entity.description.strip() if entity.description is not None else None,
                 created_time=entity.createdTime,
                 created_by=created_by,
                 modified_time=entity.modifiedTime,
@@ -715,7 +715,11 @@ def get_import_data_model_columns(
             columns.append(
                 ImportDataModelColumn(
                     name=column["columnName"].strip(),
-                    description=column["descriptionName"].strip(),
+                    description=(
+                        column["descriptionName"].strip()
+                        if "descriptionName" in column and column["descriptionName"] is not None
+                        else None
+                    ),
                     property_type=column["propertyType"],
                     data_type=column["columnDataType"],
                     max_length=column.get("maxLength"),

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py b/metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py
@@ -8,7 +8,7 @@ class ResourceModel:
     namespace: str
     model_id: str
     name: str
-    description: str
+    description: Optional[str]
     system_type: Optional[str]
     connection_id: Optional[str]
     external_id: Optional[str]
@@ -22,7 +22,7 @@ class Resource:
     resource_subtype: str
     story_id: str
     name: str
-    description: str
+    description: Optional[str]
     created_time: datetime
     created_by: Optional[str]
     modified_time: datetime
@@ -36,7 +36,7 @@ class Resource:
 @dataclass(frozen=True)
 class ImportDataModelColumn:
     name: str
-    description: str
+    description: Optional[str]
     property_type: str
     data_type: str
     max_length: Optional[int]