From e546af52499a5d790a1abd7a0157438d4d7ad284 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 8 Aug 2016 16:43:04 -0700 Subject: [PATCH 1/4] fix. --- .../scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala | 1 + .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 2 ++ 2 files changed, 3 insertions(+) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 3bf4ed5ab45a..ed0958635d03 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -414,6 +414,7 @@ private[hive] class HiveClientImpl( .map(_.asScala.toMap).orNull ), properties = properties, + comment = properties.get("comment"), viewOriginalText = Option(h.getViewOriginalText), viewText = Option(h.getViewExpandedText), unsupportedFeatures = unsupportedFeatures) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index e078b585420f..9cbbdd36ea23 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -135,7 +135,9 @@ class HiveDDLSuite sql(s"CREATE VIEW $viewName COMMENT 'no comment' AS SELECT * FROM $tabName") val tableMetadata = catalog.getTableMetadata(TableIdentifier(tabName, Some("default"))) val viewMetadata = catalog.getTableMetadata(TableIdentifier(viewName, Some("default"))) + assert(tableMetadata.comment == Option("BLABLA")) assert(tableMetadata.properties.get("comment") == Option("BLABLA")) + assert(viewMetadata.comment == Option("no comment")) assert(viewMetadata.properties.get("comment") == Option("no comment")) } } From 6ed33747fb66a68fe72bfddd97e51eaa66fd5e70 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Tue, 9 Aug 2016 23:03:50 -0700 Subject: [PATCH 2/4] fix. --- .../scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala | 2 +- .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index ed0958635d03..815b14d18fa7 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -413,7 +413,7 @@ private[hive] class HiveClientImpl( properties = Option(h.getTTable.getSd.getSerdeInfo.getParameters) .map(_.asScala.toMap).orNull ), - properties = properties, + properties = properties.filterKeys(_ != "comment"), comment = properties.get("comment"), viewOriginalText = Option(h.getViewOriginalText), viewText = Option(h.getViewExpandedText), diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 9cbbdd36ea23..6e3d27a6404e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -136,9 +136,7 @@ class HiveDDLSuite val tableMetadata = catalog.getTableMetadata(TableIdentifier(tabName, Some("default"))) val viewMetadata = catalog.getTableMetadata(TableIdentifier(viewName, Some("default"))) assert(tableMetadata.comment == Option("BLABLA")) - assert(tableMetadata.properties.get("comment") == Option("BLABLA")) assert(viewMetadata.comment == Option("no comment")) - assert(viewMetadata.properties.get("comment") == Option("no comment")) } } } From e575700bad6a59a80e71cdbd3213cff0248d6256 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Tue, 9 Aug 2016 23:30:30 -0700 Subject: [PATCH 3/4] replace filterKey by filter --- .../scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 815b14d18fa7..f6fe4d62f95f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -413,7 +413,7 @@ private[hive] class HiveClientImpl( properties = Option(h.getTTable.getSd.getSerdeInfo.getParameters) .map(_.asScala.toMap).orNull ), - properties = properties.filterKeys(_ != "comment"), + properties = properties.filter(kv => kv._1 != "path"), comment = properties.get("comment"), viewOriginalText = Option(h.getViewOriginalText), viewText = Option(h.getViewExpandedText), From b1cbf55b916ddd15198285ebaec5d3da2e637ef9 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Wed, 10 Aug 2016 00:39:32 -0700 Subject: [PATCH 4/4] address comments --- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 2 +- .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index f6fe4d62f95f..f8204e183f03 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -413,7 +413,7 @@ private[hive] class HiveClientImpl( properties = Option(h.getTTable.getSd.getSerdeInfo.getParameters) .map(_.asScala.toMap).orNull ), - properties = properties.filter(kv => kv._1 != "path"), + properties = properties.filter(kv => kv._1 != "comment"), comment = properties.get("comment"), viewOriginalText = Option(h.getViewOriginalText), viewText = Option(h.getViewExpandedText), diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 6e3d27a6404e..970b6885f625 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -137,6 +137,9 @@ class HiveDDLSuite val viewMetadata = catalog.getTableMetadata(TableIdentifier(viewName, Some("default"))) assert(tableMetadata.comment == Option("BLABLA")) assert(viewMetadata.comment == Option("no comment")) + // Ensure that `comment` is removed from the table property + assert(tableMetadata.properties.get("comment").isEmpty) + assert(viewMetadata.properties.get("comment").isEmpty) } } }