Skip to content

Commit a95c3e2

Browse files
cxzl25gatorsmile
authored andcommitted
[SPARK-23230][SQL][BRANCH-2.2] When hive.default.fileformat is other kinds of file types, create textfile table cause a serde error
When hive.default.fileformat is other kinds of file types, create textfile table cause a serde error. We should take the default type of textfile and sequencefile both as org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe. ``` set hive.default.fileformat=orc; create table tbl( i string ) stored as textfile; desc formatted tbl; Serde Library org.apache.hadoop.hive.ql.io.orc.OrcSerde InputFormat org.apache.hadoop.mapred.TextInputFormat OutputFormat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat ``` Author: sychen <sychen@ctrip.com> Closes #20593 from cxzl25/default_serde_2.2.
1 parent 73263b2 commit a95c3e2

File tree

2 files changed

+23
-2
lines changed

2 files changed

+23
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ object HiveSerDe {
3131
"sequencefile" ->
3232
HiveSerDe(
3333
inputFormat = Option("org.apache.hadoop.mapred.SequenceFileInputFormat"),
34-
outputFormat = Option("org.apache.hadoop.mapred.SequenceFileOutputFormat")),
34+
outputFormat = Option("org.apache.hadoop.mapred.SequenceFileOutputFormat"),
35+
serde = Option("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")),
3536

3637
"rcfile" ->
3738
HiveSerDe(
@@ -54,7 +55,8 @@ object HiveSerDe {
5455
"textfile" ->
5556
HiveSerDe(
5657
inputFormat = Option("org.apache.hadoop.mapred.TextInputFormat"),
57-
outputFormat = Option("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat")),
58+
outputFormat = Option("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"),
59+
serde = Option("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")),
5860

5961
"avro" ->
6062
HiveSerDe(

sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,25 @@ class HiveDDLCommandSuite extends PlanTest with SQLTestUtils with TestHiveSingle
615615
assert(output == Some("org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"))
616616
assert(serde == Some("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"))
617617
}
618+
619+
withSQLConf("hive.default.fileformat" -> "orc") {
620+
val (desc, exists) = extractTableDesc(
621+
"CREATE TABLE IF NOT EXISTS fileformat_test (id int) STORED AS textfile")
622+
assert(exists)
623+
assert(desc.storage.inputFormat == Some("org.apache.hadoop.mapred.TextInputFormat"))
624+
assert(desc.storage.outputFormat ==
625+
Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"))
626+
assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
627+
}
628+
629+
withSQLConf("hive.default.fileformat" -> "orc") {
630+
val (desc, exists) = extractTableDesc(
631+
"CREATE TABLE IF NOT EXISTS fileformat_test (id int) STORED AS sequencefile")
632+
assert(exists)
633+
assert(desc.storage.inputFormat == Some("org.apache.hadoop.mapred.SequenceFileInputFormat"))
634+
assert(desc.storage.outputFormat == Some("org.apache.hadoop.mapred.SequenceFileOutputFormat"))
635+
assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
636+
}
618637
}
619638

620639
test("table name with schema") {

0 commit comments

Comments
 (0)