From 13f6054a76f42a9ab22017ddeb093d7172e06cb0 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Thu, 26 Mar 2020 12:56:08 +0300 Subject: [PATCH 1/2] Migrate `DateTimeTestUtils` from `TimeZone` to `ZoneId` --- .../org/apache/spark/sql/avro/AvroSuite.scala | 4 +- .../sql/catalyst/expressions/CastSuite.scala | 9 +-- .../CollectionExpressionsSuite.scala | 4 +- .../expressions/CsvExpressionsSuite.scala | 10 +-- .../expressions/DateExpressionsSuite.scala | 63 ++++++++++--------- .../expressions/JsonExpressionsSuite.scala | 8 +-- .../parser/ExpressionParserSuite.scala | 6 +- .../sql/catalyst/util/DateTimeTestUtils.scala | 15 ++--- .../catalyst/util/DateTimeUtilsSuite.scala | 52 ++++++++------- .../spark/sql/DataFrameFunctionsSuite.scala | 2 +- .../spark/sql/StatisticsCollectionSuite.scala | 3 +- 11 files changed, 90 insertions(+), 86 deletions(-) diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index 34a0e2bf607a9..a04037c17d7e6 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -38,7 +38,7 @@ import org.apache.spark.sql._ import org.apache.spark.sql.TestingUDT.IntervalData import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.catalyst.plans.logical.Filter -import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils} +import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, UTC} import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.datasources.{DataSource, FilePartition} import org.apache.spark.sql.execution.datasources.v2.BatchScanExec @@ -408,7 +408,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession { StructField("float", FloatType, true), StructField("date", DateType, true) )) - DateTimeTestUtils.withDefaultTimeZone(DateTimeUtils.TimeZoneUTC) { + withDefaultTimeZone(UTC) { val rdd = spark.sparkContext.parallelize(Seq( Row(1f, null), Row(2f, new Date(1451948400000L)), diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index 79c4eb1222831..89e15147c61ad 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -111,13 +111,14 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper { } test("cast string to timestamp") { - new ParVector(ALL_TIMEZONES.toVector).foreach { tz => + new ParVector(ALL_TIMEZONES.toVector).foreach { zid => def checkCastStringToTimestamp(str: String, expected: Timestamp): Unit = { - checkEvaluation(cast(Literal(str), TimestampType, Option(tz.getID)), expected) + checkEvaluation(cast(Literal(str), TimestampType, Option(zid.getId)), expected) } checkCastStringToTimestamp("123", null) + val tz = TimeZone.getTimeZone(zid) var c = Calendar.getInstance(tz) c.set(2015, 0, 1, 0, 0, 0) c.set(Calendar.MILLISECOND, 0) @@ -262,10 +263,10 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper { val zts = sd + " 00:00:00" val sts = sd + " 00:00:02" val nts = sts + ".1" - val ts = withDefaultTimeZone(TimeZoneGMT)(Timestamp.valueOf(nts)) + val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts)) for (tz <- ALL_TIMEZONES) { - val timeZoneId = Option(tz.getID) + val timeZoneId = Option(tz.getId) var c = Calendar.getInstance(TimeZoneGMT) c.set(2015, 2, 8, 2, 30, 0) checkEvaluation( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index 44b90b772b38a..a4baad0c72444 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -852,7 +852,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper DateTimeUtils.millisToMicros(Timestamp.valueOf(s).getTime - offset) } - DateTimeTestUtils.withDefaultTimeZone(timeZone) { + DateTimeTestUtils.withDefaultTimeZone(timeZone.toZoneId) { // Spring time change checkEvaluation(new Sequence( Literal(Timestamp.valueOf("2018-03-25 01:30:00")), @@ -880,7 +880,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper } test("Sequence of dates") { - DateTimeTestUtils.withDefaultTimeZone(TimeZone.getTimeZone(UTC)) { + DateTimeTestUtils.withDefaultTimeZone(UTC) { checkEvaluation(new Sequence( Literal(Date.valueOf("2018-01-01")), Literal(Date.valueOf("2018-01-05")), diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala index 24cf9339aa72e..16d78b0526503 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.catalyst.expressions import java.text.SimpleDateFormat -import java.util.{Calendar, Locale} +import java.util.{Calendar, Locale, TimeZone} import org.scalatest.exceptions.TestFailedException @@ -89,8 +89,8 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P ) val csvData2 = "2016-01-01T00:00:00" - for (tz <- DateTimeTestUtils.outstandingTimezones) { - c = Calendar.getInstance(tz) + for (zid <- DateTimeTestUtils.outstandingZoneIds) { + c = Calendar.getInstance(TimeZone.getTimeZone(zid)) c.set(2016, 0, 1, 0, 0, 0) c.set(Calendar.MILLISECOND, 0) checkEvaluation( @@ -98,14 +98,14 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P schema, Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss"), Literal(csvData2), - Option(tz.getID)), + Option(zid.getId)), InternalRow(c.getTimeInMillis * 1000L) ) checkEvaluation( CsvToStructs( schema, Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss", - DateTimeUtils.TIMEZONE_OPTION -> tz.getID), + DateTimeUtils.TIMEZONE_OPTION -> zid.getId), Literal(csvData2), UTC_OPT), InternalRow(c.getTimeInMillis * 1000L) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 8049958ba72e7..024276355edbc 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -185,9 +185,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(Second(Literal(ts), UTC_OPT), 15) val c = Calendar.getInstance() - for (tz <- outstandingTimezones) { - val timeZoneId = Option(tz.getID) - c.setTimeZone(tz) + for (zid <- outstandingZoneIds) { + val timeZoneId = Option(zid.getId) + c.setTimeZone(TimeZone.getTimeZone(zid)) (0 to 60 by 5).foreach { s => c.set(2015, 18, 3, 3, 5, s) checkEvaluation( @@ -285,9 +285,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(Hour(Literal(ts), UTC_OPT), 13) val c = Calendar.getInstance() - for (tz <- outstandingTimezones) { - val timeZoneId = Option(tz.getID) - c.setTimeZone(tz) + for (zid <- outstandingZoneIds) { + val timeZoneId = Option(zid.getId) + c.setTimeZone(TimeZone.getTimeZone(zid)) (0 to 24 by 6).foreach { h => (0 to 60 by 30).foreach { m => (0 to 60 by 30).foreach { s => @@ -312,9 +312,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(Minute(Literal(ts), UTC_OPT), 10) val c = Calendar.getInstance() - for (tz <- outstandingTimezones) { - val timeZoneId = Option(tz.getID) - c.setTimeZone(tz) + for (zid <- outstandingZoneIds) { + val timeZoneId = Option(zid.getId) + c.setTimeZone(TimeZone.getTimeZone(zid)) (0 to 60 by 5).foreach { m => (0 to 60 by 15).foreach { s => c.set(2015, 18, 3, 3, m, s) @@ -384,9 +384,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { test("time_add") { val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US) - for (tz <- outstandingTimezones) { - val timeZoneId = Option(tz.getID) - sdf.setTimeZone(tz) + for (zid <- outstandingZoneIds) { + val timeZoneId = Option(zid.getId) + sdf.setTimeZone(TimeZone.getTimeZone(zid)) checkEvaluation( TimeAdd( @@ -422,9 +422,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { test("time_sub") { val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US) - for (tz <- outstandingTimezones) { - val timeZoneId = Option(tz.getID) - sdf.setTimeZone(tz) + for (zid <- outstandingZoneIds) { + val timeZoneId = Option(zid.getId) + sdf.setTimeZone(TimeZone.getTimeZone(zid)) checkEvaluation( TimeSub( @@ -508,9 +508,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { test("months_between") { val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) - for (tz <- outstandingTimezones) { - val timeZoneId = Option(tz.getID) - sdf.setTimeZone(tz) + for (zid <- outstandingZoneIds) { + val timeZoneId = Option(zid.getId) + sdf.setTimeZone(TimeZone.getTimeZone(zid)) checkEvaluation( MonthsBetween( @@ -641,7 +641,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { expected) } - withDefaultTimeZone(TimeZone.getTimeZone(UTC)) { + withDefaultTimeZone(UTC) { val inputDate = Timestamp.valueOf("2015-07-22 05:30:06") Seq("yyyy", "YYYY", "year", "YEAR", "yy", "YY").foreach { fmt => @@ -711,8 +711,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { val sdf1 = new SimpleDateFormat(fmt1, Locale.US) val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS" val sdf2 = new SimpleDateFormat(fmt2, Locale.US) - for (tz <- outstandingTimezones) { - val timeZoneId = Option(tz.getID) + for (zid <- outstandingZoneIds) { + val timeZoneId = Option(zid.getId) + val tz = TimeZone.getTimeZone(zid) sdf1.setTimeZone(tz) sdf2.setTimeZone(tz) @@ -762,9 +763,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { val sdf3 = new SimpleDateFormat(fmt3, Locale.US) sdf3.setTimeZone(TimeZone.getTimeZone(UTC)) - withDefaultTimeZone(TimeZone.getTimeZone(UTC)) { - for (tz <- outstandingTimezones) { - val timeZoneId = Option(tz.getID) + withDefaultTimeZone(UTC) { + for (zid <- outstandingZoneIds) { + val timeZoneId = Option(zid.getId) + val tz = TimeZone.getTimeZone(zid) sdf1.setTimeZone(tz) sdf2.setTimeZone(tz) @@ -829,9 +831,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { val sdf3 = new SimpleDateFormat(fmt3, Locale.US) sdf3.setTimeZone(TimeZone.getTimeZone(UTC)) - withDefaultTimeZone(TimeZone.getTimeZone(UTC)) { - for (tz <- outstandingTimezones) { - val timeZoneId = Option(tz.getID) + withDefaultTimeZone(UTC) { + for (zid <- outstandingZoneIds) { + val timeZoneId = Option(zid.getId) + val tz = TimeZone.getTimeZone(zid) sdf1.setTimeZone(tz) sdf2.setTimeZone(tz) @@ -847,7 +850,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation( ToUnixTimestamp(Literal(date1), Literal(fmt1), timeZoneId), MICROSECONDS.toSeconds( - DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), tz.toZoneId))) + DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), zid))) checkEvaluation( ToUnixTimestamp( Literal(sdf2.format(new Timestamp(-1000000))), @@ -856,7 +859,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(ToUnixTimestamp( Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId), MICROSECONDS.toSeconds(DateTimeUtils.daysToMicros( - DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz.toZoneId))) + DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), zid))) val t1 = ToUnixTimestamp( CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long] val t2 = ToUnixTimestamp( @@ -871,7 +874,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(ToUnixTimestamp( Literal(date1), Literal.create(null, StringType), timeZoneId), MICROSECONDS.toSeconds( - DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), tz.toZoneId))) + DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), zid))) checkEvaluation( ToUnixTimestamp( Literal("2015-07-24"), diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index dec2e4665251f..032e0ac61884b 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -517,8 +517,8 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with ) val jsonData2 = """{"t": "2016-01-01T00:00:00"}""" - for (tz <- DateTimeTestUtils.outstandingTimezones) { - c = Calendar.getInstance(tz) + for (zid <- DateTimeTestUtils.outstandingZoneIds) { + c = Calendar.getInstance(TimeZone.getTimeZone(zid)) c.set(2016, 0, 1, 0, 0, 0) c.set(Calendar.MILLISECOND, 0) checkEvaluation( @@ -526,14 +526,14 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with schema, Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss"), Literal(jsonData2), - Option(tz.getID)), + Option(zid.getId)), InternalRow(c.getTimeInMillis * 1000L) ) checkEvaluation( JsonToStructs( schema, Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss", - DateTimeUtils.TIMEZONE_OPTION -> tz.getID), + DateTimeUtils.TIMEZONE_OPTION -> zid.getId), Literal(jsonData2), UTC_OPT), InternalRow(c.getTimeInMillis * 1000L) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala index 74fd48d922a3a..522d49e448f1f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala @@ -772,10 +772,10 @@ class ExpressionParserSuite extends AnalysisTest { } test("timestamp literals") { - DateTimeTestUtils.outstandingTimezones.foreach { timeZone => - withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone.getID) { + DateTimeTestUtils.outstandingZoneIds.foreach { zid => + withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId) { def toMicros(time: LocalDateTime): Long = { - val seconds = time.atZone(timeZone.toZoneId).toInstant.getEpochSecond + val seconds = time.atZone(zid).toInstant.getEpochSecond TimeUnit.SECONDS.toMicros(seconds) } assertEval( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala index 374400ecdb058..bf9e8f71ba1c9 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala @@ -21,6 +21,8 @@ import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId, ZoneOffset} import java.util.TimeZone import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ + import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeUtils.getZoneId @@ -39,7 +41,7 @@ object DateTimeTestUtils { val UTC_OPT = Option("UTC") - val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone) + val ALL_TIMEZONES: Seq[ZoneId] = ZoneId.getAvailableZoneIds.asScala.map(getZoneId).toSeq val outstandingTimezonesIds: Seq[String] = Seq( "UTC", @@ -50,16 +52,15 @@ object DateTimeTestUtils { "Antarctica/Vostok", "Asia/Hong_Kong", "Europe/Amsterdam") - val outstandingTimezones: Seq[TimeZone] = outstandingTimezonesIds.map(TimeZone.getTimeZone) - val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(DateTimeUtils.getZoneId) + val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(getZoneId) - def withDefaultTimeZone[T](newDefaultTimeZone: TimeZone)(block: => T): T = { - val originalDefaultTimeZone = TimeZone.getDefault + def withDefaultTimeZone[T](newDefaultTimeZone: ZoneId)(block: => T): T = { + val originalDefaultTimeZone = ZoneId.systemDefault() try { - TimeZone.setDefault(newDefaultTimeZone) + TimeZone.setDefault(TimeZone.getTimeZone(newDefaultTimeZone)) block } finally { - TimeZone.setDefault(originalDefaultTimeZone) + TimeZone.setDefault(TimeZone.getTimeZone(originalDefaultTimeZone)) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 89101c2c493f6..17258e82838d9 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -154,11 +154,10 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { } test("string to timestamp") { - for (tz <- ALL_TIMEZONES) { + for (zid <- ALL_TIMEZONES) { def checkStringToTimestamp(str: String, expected: Option[Long]): Unit = { - assert(toTimestamp(str, tz.toZoneId) === expected) + assert(toTimestamp(str, zid) === expected) } - val zid = tz.toZoneId checkStringToTimestamp("1969-12-31 16:00:00", Option(date(1969, 12, 31, 16, zid = zid))) checkStringToTimestamp("0001", Option(date(1, 1, 1, 0, zid = zid))) @@ -465,7 +464,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { } } - withDefaultTimeZone(TimeZone.getTimeZone(LA.getId)) { + withDefaultTimeZone(LA) { // Daylight Saving Time test("2016-03-13 09:59:59.0", LA.getId, "2016-03-13 01:59:59.0") test("2016-03-13 10:00:00.0", LA.getId, "2016-03-13 03:00:00.0") @@ -481,8 +480,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { === expected) } - for (tz <- ALL_TIMEZONES) { - withDefaultTimeZone(tz) { + for (zid <- ALL_TIMEZONES) { + withDefaultTimeZone(zid) { test("2011-12-25 09:00:00.123456", "UTC", "2011-12-25 09:00:00.123456") test("2011-12-25 18:00:00.123456", JST.getId, "2011-12-25 09:00:00.123456") test("2011-12-25 01:00:00.123456", LA.getId, "2011-12-25 09:00:00.123456") @@ -491,7 +490,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { } val tz = LA.getId - withDefaultTimeZone(TimeZone.getTimeZone(tz)) { + withDefaultTimeZone(getZoneId(tz)) { // Daylight Saving Time test("2016-03-13 01:59:59", tz, "2016-03-13 09:59:59.0") test("2016-03-13 02:00:00", tz, "2016-03-13 10:00:00.0") @@ -552,9 +551,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { testTrunc(DateTimeUtils.TRUNC_TO_CENTURY, "2001-01-01", defaultInputTS.get) testTrunc(DateTimeUtils.TRUNC_TO_MILLENNIUM, "2001-01-01", defaultInputTS.get) - for (tz <- ALL_TIMEZONES) { - withDefaultTimeZone(tz) { - val zid = tz.toZoneId + for (zid <- ALL_TIMEZONES) { + withDefaultTimeZone(zid) { val inputTS = DateTimeUtils.stringToTimestamp( UTF8String.fromString("2015-03-05T09:32:05.359"), defaultZoneId) val inputTS1 = DateTimeUtils.stringToTimestamp( @@ -610,14 +608,14 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { "Pacific/Kiritimati" -> Set(9130, 9131), "Pacific/Kwajalein" -> Set(8632, 8633, 8634), MIT.getId -> Set(15338)) - for (tz <- ALL_TIMEZONES) { - val skipped = skipped_days.getOrElse(tz.getID, Set.empty) + for (zid <- ALL_TIMEZONES) { + val skipped = skipped_days.getOrElse(zid.getId, Set.empty) val testingData = Seq(-20000, 20000) ++ (1 to 1000).map(_ => (math.random() * 40000 - 20000).toInt) testingData.foreach { d => if (!skipped.contains(d)) { - assert(microsToDays(daysToMicros(d, tz.toZoneId), tz.toZoneId) === d, - s"Round trip of $d did not work in tz $tz") + assert(microsToDays(daysToMicros(d, zid), zid) === d, + s"Round trip of $d did not work in tz ${zid.getId}") } } } @@ -672,8 +670,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { } test("rebase julian to/from gregorian micros") { - outstandingTimezones.foreach { timeZone => - withDefaultTimeZone(timeZone) { + outstandingZoneIds.foreach { zid => + withDefaultTimeZone(zid) { Seq( "0001-01-01 01:02:03.654321", "1000-01-01 03:02:01.123456", @@ -684,9 +682,9 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { "1969-12-31 11:22:33.000100", "1970-01-01 00:00:00.000001", // The epoch day "2020-03-14 09:33:01.500000").foreach { ts => - withClue(s"time zone = ${timeZone.getID} ts = $ts") { + withClue(s"time zone = ${zid.getId} ts = $ts") { val julianMicros = parseToJulianMicros(ts) - val gregMicros = parseToGregMicros(ts.replace(' ', 'T'), timeZone.toZoneId) + val gregMicros = parseToGregMicros(ts.replace(' ', 'T'), zid) assert(rebaseJulianToGregorianMicros(julianMicros) === gregMicros) assert(rebaseGregorianToJulianMicros(gregMicros) === julianMicros) @@ -706,8 +704,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { } test("rebase gregorian to/from julian days") { - outstandingTimezones.foreach { timeZone => - withDefaultTimeZone(timeZone) { + outstandingZoneIds.foreach { zid => + withDefaultTimeZone(zid) { Seq( "0001-01-01", "1000-01-01", @@ -729,14 +727,14 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { } test("rebase julian to gregorian date for leap years") { - outstandingTimezones.foreach { timeZone => - withDefaultTimeZone(timeZone) { + outstandingZoneIds.foreach { zid => + withDefaultTimeZone(zid) { Seq( "1000-02-29" -> "1000-03-01", "1600-02-29" -> "1600-02-29", "1700-02-29" -> "1700-03-01", "2000-02-29" -> "2000-02-29").foreach { case (julianDate, gregDate) => - withClue(s"tz = ${timeZone.getID} julian date = $julianDate greg date = $gregDate") { + withClue(s"tz = ${zid.getId} julian date = $julianDate greg date = $gregDate") { val date = Date.valueOf(julianDate) val julianDays = fromJavaDateLegacy(date) val gregorianDays = localDateToDays(LocalDate.parse(gregDate)) @@ -749,17 +747,17 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { } test("rebase julian to gregorian timestamp for leap years") { - outstandingTimezones.foreach { timeZone => - withDefaultTimeZone(timeZone) { + outstandingZoneIds.foreach { zid => + withDefaultTimeZone(zid) { Seq( "1000-02-29 01:02:03.123456" -> "1000-03-01T01:02:03.123456", "1600-02-29 11:12:13.654321" -> "1600-02-29T11:12:13.654321", "1700-02-29 21:22:23.000001" -> "1700-03-01T21:22:23.000001", "2000-02-29 00:00:00.999999" -> "2000-02-29T00:00:00.999999" ).foreach { case (julianTs, gregTs) => - withClue(s"tz = ${timeZone.getID} julian ts = $julianTs greg ts = $gregTs") { + withClue(s"tz = ${zid.getId} julian ts = $julianTs greg ts = $gregTs") { val julianMicros = parseToJulianMicros(julianTs) - val gregorianMicros = parseToGregMicros(gregTs, timeZone.toZoneId) + val gregorianMicros = parseToGregMicros(gregTs, zid) assert(rebaseJulianToGregorianMicros(julianMicros) === gregorianMicros) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala index 875a671a01bfb..c1d8f736acdf2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala @@ -1029,7 +1029,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession { Timestamp.valueOf("2018-01-01 12:00:00"), Timestamp.valueOf("2018-01-02 00:00:00"))))) - DateTimeTestUtils.withDefaultTimeZone(TimeZone.getTimeZone(UTC)) { + DateTimeTestUtils.withDefaultTimeZone(UTC) { checkAnswer( spark.sql("select sequence(" + " cast('2018-01-01' as date)" + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala index 30b15a8710d8c..708b98e8fe15a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala @@ -481,7 +481,8 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared } } - DateTimeTestUtils.outstandingTimezones.foreach { timeZone => + DateTimeTestUtils.outstandingZoneIds.foreach { zid => + val timeZone = TimeZone.getTimeZone(zid) checkTimestampStats(DateType, DateTimeUtils.TimeZoneUTC, timeZone) { stats => assert(stats.min.get.asInstanceOf[Int] == TimeUnit.SECONDS.toDays(start)) assert(stats.max.get.asInstanceOf[Int] == TimeUnit.SECONDS.toDays(end - 1)) From 7d2b5bd3d981ec48b280b60ea8acd525b7fb8a5a Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Fri, 27 Mar 2020 09:25:56 +0300 Subject: [PATCH 2/2] Address Wenchen's review comment --- .../apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 17258e82838d9..6a164311a447c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -489,8 +489,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { } } - val tz = LA.getId - withDefaultTimeZone(getZoneId(tz)) { + withDefaultTimeZone(LA) { + val tz = LA.getId // Daylight Saving Time test("2016-03-13 01:59:59", tz, "2016-03-13 09:59:59.0") test("2016-03-13 02:00:00", tz, "2016-03-13 10:00:00.0")