Skip to content

Commit 06be013

Browse files
committed
Remove nested getSchema() in UnivocityParserSuite
1 parent df30439 commit 06be013

File tree

1 file changed

+13
-18
lines changed

1 file changed

+13
-18
lines changed

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.InternalRow
2828
import org.apache.spark.sql.catalyst.plans.SQLHelper
2929
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
3030
import org.apache.spark.sql.catalyst.util.DateTimeUtils
31-
import org.apache.spark.sql.sources.{EqualTo, Filter, LessThan, StringStartsWith}
31+
import org.apache.spark.sql.sources.{EqualTo, Filter, StringStartsWith}
3232
import org.apache.spark.sql.types._
3333
import org.apache.spark.unsafe.types.UTF8String
3434

@@ -273,21 +273,13 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
273273
test("skipping rows using pushdown filters") {
274274
def check(
275275
input: String = "1,a",
276-
dataSchema: String = "i INTEGER, s STRING",
277-
requiredSchema: String = "i INTEGER",
276+
dataSchema: StructType = StructType.fromDDL("i INTEGER, s STRING"),
277+
requiredSchema: StructType = StructType.fromDDL("i INTEGER"),
278278
filters: Seq[Filter],
279279
expected: Seq[InternalRow]): Unit = {
280-
def getSchema(str: String): StructType = str match {
281-
case "" => new StructType()
282-
case _ => StructType.fromDDL(str)
283-
}
284280
Seq(false, true).foreach { columnPruning =>
285281
val options = new CSVOptions(Map.empty[String, String], columnPruning, "GMT")
286-
val parser = new UnivocityParser(
287-
getSchema(dataSchema),
288-
getSchema(requiredSchema),
289-
options,
290-
filters)
282+
val parser = new UnivocityParser(dataSchema, requiredSchema, options, filters)
291283
val actual = parser.parse(input)
292284
assert(actual === expected)
293285
}
@@ -296,15 +288,18 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
296288
check(filters = Seq(), expected = Seq(InternalRow(1)))
297289
check(filters = Seq(EqualTo("i", 1)), expected = Seq(InternalRow(1)))
298290
check(filters = Seq(EqualTo("i", 2)), expected = Seq())
299-
check(requiredSchema = "s STRING", filters = Seq(StringStartsWith("s", "b")), expected = Seq())
300291
check(
301-
requiredSchema = "i INTEGER, s STRING",
292+
requiredSchema = StructType.fromDDL("s STRING"),
293+
filters = Seq(StringStartsWith("s", "b")),
294+
expected = Seq())
295+
check(
296+
requiredSchema = StructType.fromDDL("i INTEGER, s STRING"),
302297
filters = Seq(StringStartsWith("s", "a")),
303298
expected = Seq(InternalRow(1, UTF8String.fromString("a"))))
304299
check(
305300
input = "1,a,3.14",
306-
dataSchema = "i INTEGER, s STRING, d DOUBLE",
307-
requiredSchema = "i INTEGER, d DOUBLE",
301+
dataSchema = StructType.fromDDL("i INTEGER, s STRING, d DOUBLE"),
302+
requiredSchema = StructType.fromDDL("i INTEGER, d DOUBLE"),
308303
filters = Seq(EqualTo("d", 3.14)),
309304
expected = Seq(InternalRow(1, 3.14)))
310305

@@ -315,8 +310,8 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
315310

316311
val errMsg2 = intercept[IllegalArgumentException] {
317312
check(
318-
dataSchema = "",
319-
requiredSchema = "",
313+
dataSchema = new StructType(),
314+
requiredSchema = new StructType(),
320315
filters = Seq(EqualTo("i", 1)),
321316
expected = Seq(InternalRow.empty))
322317
}.getMessage

0 commit comments

Comments
 (0)