forbid mixed patterns in formatter

Author: yaooqinn

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala

@@ -23,8 +23,6 @@ import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverSt
 import java.time.temporal._
 import java.util.Locale
 
-import scala.util.control.NonFatal
-
 import com.google.common.cache.CacheBuilder
 
 import org.apache.spark.SparkUpgradeException
@@ -33,91 +31,51 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 
 trait DateTimeFormatterHelper {
-  private def getFieldValue(
-      accessor: TemporalAccessor,
-      field: TemporalField): Option[Int] = {
+
+  private def getOrDefault(accessor: TemporalAccessor, field: TemporalField, default: Int): Int = {
     if (accessor.isSupported(field)) {
-      try {
-        Option(accessor.get(field))
-      } catch {
-        case NonFatal(_) => None
-      }
+      accessor.get(field)
     } else {
-      None
+      default
     }
   }
 
+  private def mayNonWeekBased(accessor: TemporalAccessor): Boolean = {
+    val has = accessor.isSupported _
+    has(ChronoField.YEAR) || has(ChronoField.MONTH_OF_YEAR) || has(ChronoField.DAY_OF_MONTH)
+  }
+
+  private def mayWeekBased(accessor: TemporalAccessor, wf: WeekFields): Boolean = {
+    val has = accessor.isSupported _
+    has(wf.weekBasedYear) || has(wf.weekOfMonth) || has(wf.weekOfWeekBasedYear) || has(wf.dayOfWeek)
+  }
+
   @throws[DateTimeException]
   protected def toLocalDate(accessor: TemporalAccessor, locale: Locale): LocalDate = {
     val localDate = accessor.query(TemporalQueries.localDate())
     // If all the date fields are resolved(yMd or Ywu), return the local date directly.
     if (localDate != null) return localDate
 
-    var res = LocalDate.of(1970, 1, 1)
-
     val weekFields = WeekFields.of(locale)
-    val weekBasedYearField = weekFields.weekBasedYear
-    var weekBasedYearEnabled = false
-
-    val year = getFieldValue(accessor, weekBasedYearField).map { y =>
-      weekBasedYearEnabled = true
-      y
-    }.orElse {
-      getFieldValue(accessor, ChronoField.YEAR)
-    }
-
-    val week = getFieldValue(accessor, weekFields.weekOfWeekBasedYear())
-    val dayOfWeek = getFieldValue(accessor, weekFields.dayOfWeek())
-
-    // TODO: How to check 'W' week-of-month field, not like other week-based field, it always throw
-    // UnsupportedTemporalTypeException to get it even `accessor.isSupported` passed.
-
-    if (weekBasedYearEnabled) {
-      // If the week-based-year field exists, only the week-based fields matters.
-      res.`with`(weekFields.weekOfWeekBasedYear, week.getOrElse(1).toLong)
-        .`with`(weekFields.dayOfWeek(), dayOfWeek.getOrElse(1).toLong)
-        .`with`(weekBasedYearField, year.get)
+    val weekBased = mayWeekBased(accessor, weekFields)
+    if (weekBased && mayNonWeekBased(accessor)) {
+      throw new DateTimeException(
+        s"Can not mix week-based and non-week-based date fields together for parsing dates")
+    } else if (weekBased) {
+
+      val weekBasedYear = getOrDefault(accessor, weekFields.weekBasedYear, 1970)
+      val week = getOrDefault(accessor, weekFields.weekOfWeekBasedYear(), 1)
+      val dayOfWeek = getOrDefault(accessor, weekFields.dayOfWeek(), 1)
+
+      LocalDate.of(1970, 1, 1)
+        .`with`(weekFields.weekOfWeekBasedYear, week)
+        .`with`(weekFields.dayOfWeek, dayOfWeek)
+        .`with`(weekFields.weekBasedYear, weekBasedYear)
     } else {
-      if (year.isDefined) {
-        res = res.withYear(year.get)
-      }
-
-      val month = getFieldValue(accessor, ChronoField.MONTH_OF_YEAR)
-      val day = getFieldValue(accessor, ChronoField.DAY_OF_MONTH)
-
-      if (month.isDefined && week.isDefined) {
-        // check the week fall into the correct month of the year
-        res = res.`with`(weekFields.weekOfWeekBasedYear(), week.get)
-        if (res.getMonthValue != month.get) {
-          throw new DateTimeException(
-            s"week-of-week-based-year value: ${week.get} conflicts with month-of-year value:" +
-              s" ${month.get} which should be ${res.getMonthValue} instead.")
-        }
-      } else if (month.isDefined) {
-        res = res.withMonth(month.get)
-      } else if (week.isDefined) {
-        if (day.isDefined) {
-          throw new DateTimeException(
-            s"Can not use week-of-week-based-year and day-of-month together in non-week-based" +
-              s" mode.")
-        }
-        res = res.`with`(weekFields.weekOfWeekBasedYear, week.get)
-      }
-
-      if (dayOfWeek.isDefined && day.isDefined) {
-        // check whether the days matches
-        res = res.`with`(weekFields.dayOfWeek(), dayOfWeek.get)
-        if (res.getDayOfMonth != day.get) {
-          throw new DateTimeException(
-            s"day-of-week value: ${dayOfWeek.get} conflicts with day-of-month value:" +
-              s" ${day.get} which should be ${res.getDayOfMonth} instead.")
-        }
-      } else if (day.isDefined) {
-        res = res.withDayOfMonth(day.get)
-      } else if (dayOfWeek.isDefined) {
-        res = res.`with`(weekFields.dayOfWeek(), dayOfWeek.get)
-      }
-      res
+      val year = getOrDefault(accessor, ChronoField.YEAR, 1970)
+      val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
+      val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
+      LocalDate.of(year, month, day)
     }
   }
 
@@ -134,10 +92,10 @@ trait DateTimeFormatterHelper {
     } else {
       0
     }
-    val minute = getFieldValue(accessor, ChronoField.MINUTE_OF_HOUR)
-    val second = getFieldValue(accessor, ChronoField.SECOND_OF_MINUTE)
-    val nanoSecond = getFieldValue(accessor, ChronoField.NANO_OF_SECOND)
-    LocalTime.of(hour, minute.getOrElse(0), second.getOrElse(0), nanoSecond.getOrElse(0))
+    val minute = getOrDefault(accessor, ChronoField.MINUTE_OF_HOUR, 0)
+    val second = getOrDefault(accessor, ChronoField.SECOND_OF_MINUTE, 0)
+    val nanoSecond = getOrDefault(accessor, ChronoField.NANO_OF_SECOND, 0)
+    LocalTime.of(hour, minute, second, nanoSecond)
   }
 
   // Converts the parsed temporal object to ZonedDateTime. It sets time components to zeros

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala

@@ -1170,63 +1170,41 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("SPARK-31868: Restore the behaviour week-based-year for 2.4") {
-    // TODO: Locale.US is Sunday started day of week, which affects the new formatter,
-    // while in 2.4 it's Monday first.
-    checkEvaluation(
-      new ParseToTimestamp(Literal("2018-11-17"), Literal("YYYY-MM-dd")).child,
-      Timestamp.valueOf("2017-12-31 00:00:00.0"))
-
-    checkEvaluation(
-      new ParseToTimestamp(Literal("2018-11-17 13:33:33"), Literal("YYYY-MM-dd HH:mm:ss")).child,
-      Timestamp.valueOf("2017-12-31 13:33:33.0"))
-
     checkEvaluation(
       new ParseToTimestamp(Literal("1969 1 2"), Literal("YYYY w u")).child,
       Timestamp.valueOf("1968-12-30 00:00:00.0"))
 
     // the existence of 'W' is not for generating the timestamp but likely for checking whether
     // the timestamp falling into it.
     checkEvaluation(
-      new ParseToTimestamp(Literal("1969 5 1 2"), Literal("YYYY W w u")).child,
+      new ParseToTimestamp(Literal("1969 1 2"), Literal("YYYY w u")).child,
       Timestamp.valueOf("1968-12-30 00:00:00.0"))
 
     checkEvaluation(
       new ParseToTimestamp(Literal("1969 5 2"), Literal("YYYY W u")).child,
       Timestamp.valueOf("1968-12-30 00:00:00.0"))
 
-//    // the legacy parser does not support 'W' and results null, so SparkUpgradeException will come
-//    checkExceptionInExpression[SparkUpgradeException](
-//      new ParseToTimestamp(Literal("1969 4 2"), Literal("YYYY W u")).child, "3.0")
-//    checkExceptionInExpression[SparkUpgradeException](
-//      new ParseToTimestamp(Literal("5"), Literal("W")).child, "3.0")
-
+    checkEvaluation(
+      new ParseToTimestamp(Literal("2018-11-2-17"), Literal("yyyy-ww-dd")).child,
+      null)
     // https://bugs.openjdk.java.net/browse/JDK-8145633
     // Adjacent value parsing not supported for Localized Patterns
     checkExceptionInExpression[SparkUpgradeException](
       new ParseToTimestamp(Literal("196940"), Literal("YYYYww")).child, "3.0")
 
-    checkEvaluation(
-      new ParseToTimestamp(Literal("2020 1 3 2"), Literal("yyyy M w u")).child,
-      Timestamp.valueOf("2020-01-13 00:00:00.0"))
+    checkExceptionInExpression[SparkUpgradeException](
+      new ParseToTimestamp(Literal("2018-11-17 13:33:33"), Literal("YYYY-MM-dd HH:mm:ss")).child,
+      "3.0")
 
     checkEvaluation(
       new ParseToTimestamp(Literal("2018-46-7 13:33:33"), Literal("YYYY-ww-u HH:mm:ss")).child,
       Timestamp.valueOf("2018-11-17 13:33:33.0"))
 
-    checkEvaluation(
-      new ParseToTimestamp(Literal("2018-11-17"), Literal("YYYY-ww-dd")).child,
-      Timestamp.valueOf("2018-03-11 00:00:00.0"))
-
-    checkEvaluation(
-      new ParseToTimestamp(Literal("2018-11-2-17"), Literal("yyyy-ww-dd")).child,
-      null)
+    checkExceptionInExpression[SparkUpgradeException](
+      new ParseToTimestamp(Literal("2018-11-17"), Literal("YYYY-ww-dd")).child, "3.0")
 
-    // problem of first day of week change
     checkExceptionInExpression[SparkUpgradeException](
       new ParseToTimestamp(Literal("1969 1 6 1"), Literal("yyyy M d u")).child, "3.0")
 
-    checkEvaluation(
-      new ParseToTimestamp(Literal("1969 5 11 11"), Literal("YYYY M ww dd")).child,
-      Timestamp.valueOf("1969-03-09 00:00:00.0"))
   }
 }

sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out

@@ -1030,73 +1030,81 @@ You may get a different result due to the upgrading of Spark 3.0: Fail to recogn
 -- !query
 select to_timestamp('2018-11-17 13:33:33', 'YYYY-MM-dd HH:mm:ss')
 -- !query schema
-struct<to_timestamp(2018-11-17 13:33:33, YYYY-MM-dd HH:mm:ss):timestamp>
+struct<>
 -- !query output
-2017-12-31 13:33:33
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2018-11-17 13:33:33' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
 select to_timestamp('1969-01-01', 'YYYY-MM-dd')
 -- !query schema
-struct<to_timestamp(1969-01-01, YYYY-MM-dd):timestamp>
+struct<>
 -- !query output
-1968-12-29 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1969-01-01' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
 select to_timestamp('1969-12-31', 'YYYY-MM-dd')
 -- !query schema
-struct<to_timestamp(1969-12-31, YYYY-MM-dd):timestamp>
+struct<>
 -- !query output
-1968-12-29 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1969-12-31' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
 select to_timestamp('2018-01-01', 'YYYY-MM-dd')
 -- !query schema
-struct<to_timestamp(2018-01-01, YYYY-MM-dd):timestamp>
+struct<>
 -- !query output
-2017-12-31 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2018-01-01' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
 select to_timestamp('2018-11-17 13:33:33', 'YYYY-MM-dd HH:mm:ss')
 -- !query schema
-struct<to_timestamp(2018-11-17 13:33:33, YYYY-MM-dd HH:mm:ss):timestamp>
+struct<>
 -- !query output
-2017-12-31 13:33:33
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2018-11-17 13:33:33' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
 select to_timestamp('1969 1 1', 'yyyy w u')
 -- !query schema
 struct<to_timestamp(1969 1 1, yyyy w u):timestamp>
 -- !query output
-1968-12-29 00:00:00
+NULL
 
 
 -- !query
 select to_timestamp('1969 1 1', 'yyyy M u')
 -- !query schema
-struct<to_timestamp(1969 1 1, yyyy M u):timestamp>
+struct<>
 -- !query output
-1968-12-29 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1969 1 1' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
 select to_timestamp('1 1969 1', 'M YYYY u')
 -- !query schema
-struct<to_timestamp(1 1969 1, M YYYY u):timestamp>
+struct<>
 -- !query output
-1968-12-29 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1 1969 1' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
 select to_timestamp('1969 1 1', 'YYYY M u')
 -- !query schema
-struct<to_timestamp(1969 1 1, YYYY M u):timestamp>
+struct<>
 -- !query output
-1968-12-29 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1969 1 1' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
@@ -1119,33 +1127,36 @@ struct<to_timestamp(1969 1 5 1, yyyy M d u):timestamp>
 -- !query
 select to_timestamp('2018 11 17', 'YYYY ww dd')
 -- !query schema
-struct<to_timestamp(2018 11 17, YYYY ww dd):timestamp>
+struct<>
 -- !query output
-2018-03-11 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2018 11 17' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
 select to_timestamp('1969 2 11', 'YYYY W dd')
 -- !query schema
 struct<to_timestamp(1969 2 11, YYYY W dd):timestamp>
 -- !query output
-1968-12-29 00:00:00
+NULL
 
 
 -- !query
 select to_timestamp('1969 5 11 11', 'YYYY M ww dd')
 -- !query schema
-struct<to_timestamp(1969 5 11 11, YYYY M ww dd):timestamp>
+struct<>
 -- !query output
-1969-03-09 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1969 5 11 11' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
 select to_timestamp('2020 1 3 2', 'yyyy M w u')
 -- !query schema
-struct<to_timestamp(2020 1 3 2, yyyy M w u):timestamp>
+struct<>
 -- !query output
-2020-01-13 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2020 1 3 2' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
@@ -1208,14 +1219,16 @@ NULL
 -- !query
 select to_timestamp('2020 2020', 'YYYY yyyy')
 -- !query schema
-struct<to_timestamp(2020 2020, YYYY yyyy):timestamp>
+struct<>
 -- !query output
-2019-12-29 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2020 2020' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
 
 
 -- !query
 select to_timestamp('2020 2020', 'YYYY yyyy')
 -- !query schema
-struct<to_timestamp(2020 2020, YYYY yyyy):timestamp>
+struct<>
 -- !query output
-2019-12-29 00:00:00
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2020 2020' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.