fix(ingestion/glue): fix to ingest the comment for partition key as d…

…escription (datahub-project#10189)
sleeperdeep · Jun 25, 2024 · b437793 · b437793
1 parent 649b7ae
commit b437793
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 2 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
@@ -1168,6 +1168,7 @@ def get_schema_metadata() -> Optional[SchemaMetadata]:
                 schema_fields = get_schema_fields_for_hive_column(
                     hive_column_name=partition_key["Name"],
                     hive_column_type=partition_key.get("Type", "unknown"),
+                    description=partition_key.get("Comment"),
                     default_nullable=False,
                 )
                 assert schema_fields

diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json
@@ -113,6 +113,7 @@
                             {
                                 "fieldPath": "[version=2.0].[type=int].yr",
                                 "nullable": true,
+                                "description": "test comment",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.NumberType": {}
@@ -204,6 +205,7 @@
                             {
                                 "fieldPath": "[version=2.0].[type=string].year",
                                 "nullable": true,
+                                "description": "partition test comment",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.StringType": {}

diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json
@@ -115,6 +115,7 @@
                             {
                                 "fieldPath": "[version=2.0].[type=int].yr",
                                 "nullable": true,
+                                "description": "test comment",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.NumberType": {}
@@ -206,6 +207,7 @@
                             {
                                 "fieldPath": "[version=2.0].[type=string].year",
                                 "nullable": true,
+                                "description": "partition test comment",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.StringType": {}

diff --git a/metadata-ingestion/tests/unit/test_glue_source_stubs.py b/metadata-ingestion/tests/unit/test_glue_source_stubs.py
@@ -92,7 +92,7 @@
         "Retention": 0,
         "StorageDescriptor": {
             "Columns": [
-                {"Name": "yr", "Type": "int"},
+                {"Name": "yr", "Type": "int", "Comment": "test comment"},
                 {"Name": "flightdate", "Type": "string"},
                 {"Name": "uniquecarrier", "Type": "string"},
                 {"Name": "airlineid", "Type": "int"},
@@ -129,7 +129,9 @@
             },
             "StoredAsSubDirectories": False,
         },
-        "PartitionKeys": [{"Name": "year", "Type": "string"}],
+        "PartitionKeys": [
+            {"Name": "year", "Type": "string", "Comment": "partition test comment"}
+        ],
         "TableType": "EXTERNAL_TABLE",
         "Parameters": {
             "CrawlerSchemaDeserializerVersion": "1.0",