From 0e6b41e01f8c9ac7c8a262bd58f2564a64a638e6 Mon Sep 17 00:00:00 2001 From: Kuba Tyszko Date: Fri, 16 Dec 2016 13:46:43 -0800 Subject: [PATCH] SPARK-18906 SQL - parsing CSV should return null for certain numeric fields - Sargis Dudaklayan and Kuba Tyszko - Zest Finance --- .../sql/execution/datasources/csv/CSVInferSchema.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVInferSchema.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVInferSchema.scala index c63aae9d83855..526d9196c6803 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVInferSchema.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVInferSchema.scala @@ -246,9 +246,9 @@ private[csv] object CSVTypeCast { } else { castType match { case _: ByteType => datum.toByte - case _: ShortType => datum.toShort - case _: IntegerType => datum.toInt - case _: LongType => datum.toLong + case _: ShortType => Try(datum.toShort).getOrElse(null) + case _: IntegerType => Try(datum.toInt).getOrElse(null) + case _: LongType => Try(datum.toLong).getOrElse(null) case _: FloatType => datum match { case options.nanValue => Float.NaN