feat(ingest/lookml): ingest field tags (#10792)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
datahub-project · Jul 1, 2024 · 3e47496 · 3e47496
1 parent 8b4e302
commit 3e47496
Show file tree

Hide file tree

Showing 23 changed files with 641 additions and 146 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
@@ -244,6 +244,7 @@ class ViewField:
  view_name: Optional[str] = None
  is_primary_key: bool = False
  upstream_fields: List[str] = dataclasses_field(default_factory=list)
+ tags: List[str] = dataclasses_field(default_factory=list)
 
 
 @dataclass
@@ -561,21 +562,30 @@ def _get_tag_mce_for_urn(tag_urn: str) -> MetadataChangeEvent:
 
  @staticmethod
  def _get_tags_from_field_type(
- field_type: ViewFieldType, reporter: SourceReport
+ field: ViewField, reporter: SourceReport
  ) -> Optional[GlobalTagsClass]:
- if field_type in LookerUtil.type_to_tag_map:
- return GlobalTagsClass(
- tags=[
+ schema_field_tags: List[TagAssociationClass] = [
+ TagAssociationClass(tag=builder.make_tag_urn(tag_name))
+ for tag_name in field.tags
+ ]
+
+ if field.field_type in LookerUtil.type_to_tag_map:
+ schema_field_tags.extend(
+ [
  TagAssociationClass(tag=tag_name)
- for tag_name in LookerUtil.type_to_tag_map[field_type]
+ for tag_name in LookerUtil.type_to_tag_map[field.field_type]
  ]
  )
  else:
  reporter.report_warning(
  "lookml",
- f"Failed to map view field type {field_type}. Won't emit tags for it",
+ f"Failed to map view field type {field.field_type}. Won't emit tags for measure and dimension",
  )
- return None
+
+ if schema_field_tags:
+ return GlobalTagsClass(tags=schema_field_tags)
+
+ return None
 
  @staticmethod
  def get_tag_mces() -> Iterable[MetadataChangeEvent]:
@@ -602,7 +612,7 @@ def view_field_to_schema_field(
  else f"{field.field_type.value}. {field.description}"
  ),
  globalTags=(
- LookerUtil._get_tags_from_field_type(field.field_type, reporter)
+ LookerUtil._get_tags_from_field_type(field, reporter)
  if tag_measures_and_dimensions is True
  else None
  ),

diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
@@ -1090,6 +1090,7 @@ def _get_fields(
  is_primary_key=is_primary_key,
  field_type=type_cls,
  upstream_fields=upstream_fields,
+ tags=field_dict.get("tags") or [],
  )
  fields.append(field)
  return fields

diff --git a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json
@@ -78,9 +78,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -140,9 +137,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  }
@@ -178,9 +172,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -222,9 +213,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  }

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
@@ -76,9 +76,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -137,9 +134,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json
@@ -312,9 +312,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -373,9 +370,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json
@@ -312,9 +312,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -373,9 +370,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json
@@ -324,9 +324,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -386,9 +383,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json
@@ -312,9 +312,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -373,9 +370,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
@@ -312,9 +312,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -373,9 +370,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
@@ -76,9 +76,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -137,9 +134,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },

diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json
@@ -324,9 +324,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -386,9 +383,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  },
@@ -813,8 +807,8 @@
  }
 },
 {
- "entityType": "container",
- "entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
+ "entityType": "chart",
+ "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
  "changeType": "UPSERT",
  "aspectName": "status",
  "aspect": {
@@ -830,8 +824,8 @@
  }
 },
 {
- "entityType": "chart",
- "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
+ "entityType": "dashboard",
+ "entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
  "changeType": "UPSERT",
  "aspectName": "status",
  "aspect": {
@@ -847,8 +841,8 @@
  }
 },
 {
- "entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
+ "entityType": "container",
+ "entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
  "changeType": "UPSERT",
  "aspectName": "status",
  "aspect": {
@@ -864,8 +858,8 @@
  }
 },
 {
- "entityType": "dashboard",
- "entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
  "changeType": "UPSERT",
  "aspectName": "status",
  "aspect": {

diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json
@@ -76,9 +76,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  }
@@ -113,9 +110,6 @@
  },
  "nativeDataType": "string",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  }

diff --git a/metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json b/metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json
@@ -116,7 +116,7 @@
  "aspect": {
  "json": {
  "materialized": false,
- "viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
+ "viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n tags: [\"softVersion\"]\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
  "viewLanguage": "lookml"
  }
  },
@@ -338,6 +338,9 @@
  "recursive": false,
  "globalTags": {
  "tags": [
+ {
+ "tag": "urn:li:tag:softVersion"
+ },
  {
  "tag": "urn:li:tag:Dimension"
  }
@@ -484,5 +487,21 @@
  "runId": "lookml-test",
  "lastRunId": "no-run-id-provided"
  }
+},
+{
+ "entityType": "tag",
+ "entityUrn": "urn:li:tag:softVersion",
+ "changeType": "UPSERT",
+ "aspectName": "tagKey",
+ "aspect": {
+ "json": {
+ "name": "softVersion"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1586847600000,
+ "runId": "lookml-test",
+ "lastRunId": "no-run-id-provided"
+ }
 }
 ]
diff --git a/metadata-ingestion/tests/integration/lookml/expected_output.json b/metadata-ingestion/tests/integration/lookml/expected_output.json
@@ -1388,9 +1388,6 @@
  },
  "nativeDataType": "unknown",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  },
  {
@@ -1405,9 +1402,6 @@
  },
  "nativeDataType": "unknown",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  },
  {
@@ -1422,9 +1416,6 @@
  },
  "nativeDataType": "unknown",
  "recursive": false,
- "globalTags": {
- "tags": []
- },
  "isPartOfKey": false
  }
  ],