Skip to content

Commit cd18ed2

Browse files
committed
also check multiLine codepath and selecting empty schema with count
1 parent 09c986c commit cd18ed2

File tree

1 file changed

+27
-13
lines changed
  • sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv

1 file changed

+27
-13
lines changed

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1605,19 +1605,33 @@ class CSVSuite extends QueryTest with SharedSQLContext with SQLTestUtils with Te
16051605

16061606
test("SPARK-25134: check header on parsing of dataset with projection and column pruning") {
16071607
withSQLConf(SQLConf.CSV_PARSER_COLUMN_PRUNING.key -> "true") {
1608-
withTempPath { path =>
1609-
val dir = path.getAbsolutePath
1610-
Seq(("a", "b")).toDF("columnA", "columnB").write
1611-
.format("csv")
1612-
.option("header", true)
1613-
.save(dir)
1614-
checkAnswer(spark.read
1615-
.format("csv")
1616-
.option("header", true)
1617-
.option("enforceSchema", false)
1618-
.load(dir)
1619-
.select("columnA"),
1620-
Row("a"))
1608+
Seq(false, true).foreach { multiLine =>
1609+
withTempPath { path =>
1610+
val dir = path.getAbsolutePath
1611+
Seq(("a", "b")).toDF("columnA", "columnB").write
1612+
.format("csv")
1613+
.option("header", true)
1614+
.save(dir)
1615+
1616+
// schema with one column
1617+
checkAnswer(spark.read
1618+
.format("csv")
1619+
.option("header", true)
1620+
.option("enforceSchema", false)
1621+
.option("multiLine", multiLine)
1622+
.load(dir)
1623+
.select("columnA"),
1624+
Row("a"))
1625+
1626+
// empty schema
1627+
assert(spark.read
1628+
.format("csv")
1629+
.option("header", true)
1630+
.option("enforceSchema", false)
1631+
.option("multiLine", multiLine)
1632+
.load(dir)
1633+
.count() === 1L)
1634+
}
16211635
}
16221636
}
16231637
}

0 commit comments

Comments
 (0)