From ea81daf523fe21c513a0c20be462e02339586a81 Mon Sep 17 00:00:00 2001 From: wangfei Date: Wed, 4 Feb 2015 20:51:49 +0800 Subject: [PATCH 1/4] fix URISyntaxException --- .../apache/spark/sql/hive/hiveWriterContainers.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala index aae175e426ad..6d8affa0d892 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.io.{HiveFileFormatUtils, HiveOutputFormat} import org.apache.hadoop.hive.ql.plan.{PlanUtils, TableDesc} import org.apache.hadoop.io.Writable import org.apache.hadoop.mapred._ +import org.apache.hadoop.hive.common.FileUtils import org.apache.spark.mapred.SparkHadoopMapRedUtil import org.apache.spark.sql.Row @@ -212,9 +213,14 @@ private[spark] class SparkHiveDynamicPartitionWriterContainer( .zip(row.toSeq.takeRight(dynamicPartColNames.length)) .map { case (col, rawVal) => val string = if (rawVal == null) null else String.valueOf(rawVal) - s"/$col=${if (string == null || string.isEmpty) defaultPartName else string}" - } - .mkString + s"/$col=${ + if (string == null || string.isEmpty) { + defaultPartName + } else { + FileUtils.escapePathName(string) + } + }" + }.mkString def newWriter = { val newFileSinkDesc = new FileSinkDesc( From 99981776540ab520f60d55becab096c9deda94ca Mon Sep 17 00:00:00 2001 From: wangfei Date: Thu, 5 Feb 2015 08:52:17 +0800 Subject: [PATCH 2/4] added test case --- .../apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala index a6266f611c21..588685ef5947 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala @@ -663,6 +663,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "partition_date", "partition_schema1", "partition_serde_format", + "partition_special_char", "partition_type_check", "partition_varchar1", "partition_wise_fileformat4", From f8f8bb1e6caeb90834c99aa41cec0a8e8abc6974 Mon Sep 17 00:00:00 2001 From: wangfei Date: Thu, 5 Feb 2015 09:53:50 +0800 Subject: [PATCH 3/4] added test case --- .../hive/execution/HiveCompatibilitySuite.scala | 1 - .../sql/hive/execution/HiveQuerySuite.scala | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala index 588685ef5947..a6266f611c21 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala @@ -663,7 +663,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "partition_date", "partition_schema1", "partition_serde_format", - "partition_special_char", "partition_type_check", "partition_varchar1", "partition_wise_fileformat4", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index a321452cef74..c0fbac31c15f 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -854,6 +854,22 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { } } + test("SPARK-5592: get java.net.URISyntaxException when dynamic partitioning") { + sql(""" + |create table sc as select * + |from (select '2011-01-11', '2011-01-11+14:18:26' from src tablesample (1 rows) + |union all + |select '2011-01-11', '2011-01-11+15:18:26' from src tablesample (1 rows) + |union all + |select '2011-01-11', '2011-01-11+16:18:26' from src tablesample (1 rows) ) s + """.stripMargin) + sql("create table sc_part (key string) partitioned by (ts string) stored as rcfile") + sql("set hive.exec.dynamic.partition=true") + sql("set hive.exec.dynamic.partition.mode=nonstrict") + sql("insert overwrite table sc_part partition(ts) select * from sc") + sql("drop table sc_part") + } + test("Partition spec validation") { sql("DROP TABLE IF EXISTS dp_test") sql("CREATE TABLE dp_test(key INT, value STRING) PARTITIONED BY (dp INT, sp INT)") From aa55ef4998d22d4d257280e9434e34f1a61853a8 Mon Sep 17 00:00:00 2001 From: Fei Wang Date: Sat, 7 Feb 2015 09:23:05 +0800 Subject: [PATCH 4/4] comments addressed --- .../org/apache/spark/sql/hive/hiveWriterContainers.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala index 6d8affa0d892..f136e43acc8f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala @@ -213,13 +213,13 @@ private[spark] class SparkHiveDynamicPartitionWriterContainer( .zip(row.toSeq.takeRight(dynamicPartColNames.length)) .map { case (col, rawVal) => val string = if (rawVal == null) null else String.valueOf(rawVal) - s"/$col=${ + val colString = if (string == null || string.isEmpty) { defaultPartName } else { FileUtils.escapePathName(string) } - }" + s"/$col=$colString" }.mkString def newWriter = {