@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.InternalRow
2828import org .apache .spark .sql .catalyst .plans .SQLHelper
2929import org .apache .spark .sql .catalyst .util .DateTimeConstants ._
3030import org .apache .spark .sql .catalyst .util .DateTimeUtils
31- import org .apache .spark .sql .sources .{EqualTo , Filter , LessThan , StringStartsWith }
31+ import org .apache .spark .sql .sources .{EqualTo , Filter , StringStartsWith }
3232import org .apache .spark .sql .types ._
3333import org .apache .spark .unsafe .types .UTF8String
3434
@@ -273,21 +273,13 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
273273 test(" skipping rows using pushdown filters" ) {
274274 def check (
275275 input : String = " 1,a" ,
276- dataSchema : String = " i INTEGER, s STRING" ,
277- requiredSchema : String = " i INTEGER" ,
276+ dataSchema : StructType = StructType .fromDDL( " i INTEGER, s STRING" ) ,
277+ requiredSchema : StructType = StructType .fromDDL( " i INTEGER" ) ,
278278 filters : Seq [Filter ],
279279 expected : Seq [InternalRow ]): Unit = {
280- def getSchema (str : String ): StructType = str match {
281- case " " => new StructType ()
282- case _ => StructType .fromDDL(str)
283- }
284280 Seq (false , true ).foreach { columnPruning =>
285281 val options = new CSVOptions (Map .empty[String , String ], columnPruning, " GMT" )
286- val parser = new UnivocityParser (
287- getSchema(dataSchema),
288- getSchema(requiredSchema),
289- options,
290- filters)
282+ val parser = new UnivocityParser (dataSchema, requiredSchema, options, filters)
291283 val actual = parser.parse(input)
292284 assert(actual === expected)
293285 }
@@ -296,15 +288,18 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
296288 check(filters = Seq (), expected = Seq (InternalRow (1 )))
297289 check(filters = Seq (EqualTo (" i" , 1 )), expected = Seq (InternalRow (1 )))
298290 check(filters = Seq (EqualTo (" i" , 2 )), expected = Seq ())
299- check(requiredSchema = " s STRING" , filters = Seq (StringStartsWith (" s" , " b" )), expected = Seq ())
300291 check(
301- requiredSchema = " i INTEGER, s STRING" ,
292+ requiredSchema = StructType .fromDDL(" s STRING" ),
293+ filters = Seq (StringStartsWith (" s" , " b" )),
294+ expected = Seq ())
295+ check(
296+ requiredSchema = StructType .fromDDL(" i INTEGER, s STRING" ),
302297 filters = Seq (StringStartsWith (" s" , " a" )),
303298 expected = Seq (InternalRow (1 , UTF8String .fromString(" a" ))))
304299 check(
305300 input = " 1,a,3.14" ,
306- dataSchema = " i INTEGER, s STRING, d DOUBLE" ,
307- requiredSchema = " i INTEGER, d DOUBLE" ,
301+ dataSchema = StructType .fromDDL( " i INTEGER, s STRING, d DOUBLE" ) ,
302+ requiredSchema = StructType .fromDDL( " i INTEGER, d DOUBLE" ) ,
308303 filters = Seq (EqualTo (" d" , 3.14 )),
309304 expected = Seq (InternalRow (1 , 3.14 )))
310305
@@ -315,8 +310,8 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
315310
316311 val errMsg2 = intercept[IllegalArgumentException ] {
317312 check(
318- dataSchema = " " ,
319- requiredSchema = " " ,
313+ dataSchema = new StructType () ,
314+ requiredSchema = new StructType () ,
320315 filters = Seq (EqualTo (" i" , 1 )),
321316 expected = Seq (InternalRow .empty))
322317 }.getMessage
0 commit comments