From cc81650b09de527860d361d26510f443915daaae Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sun, 11 Aug 2019 22:10:37 +0500 Subject: [PATCH 01/24] Add the date_part() function --- .../catalyst/analysis/FunctionRegistry.scala | 1 + .../expressions/datetimeExpressions.scala | 32 +++++ .../resources/sql-tests/inputs/date_part.sql | 29 ++++ .../sql-tests/results/date_part.sql.out | 124 ++++++++++++++++++ 4 files changed, 186 insertions(+) create mode 100644 sql/core/src/test/resources/sql-tests/inputs/date_part.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/date_part.sql.out diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 5177f1e55829..d5728b902757 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -417,6 +417,7 @@ object FunctionRegistry { expression[TimeWindow]("window"), expression[MakeDate]("make_date"), expression[MakeTimestamp]("make_timestamp"), + expression[DatePart]("date_part"), // collection functions expression[CreateArray]("array"), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 43cef22da834..57f2b29fda9c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1828,3 +1828,35 @@ case class MakeTimestamp( override def prettyName: String = "make_timestamp" } + +case class DatePart(field: Expression, source: Expression, child: Expression) + extends RuntimeReplaceable { + + def this(field: Expression, source: Expression) { + this(field, source, { + if (!field.foldable) { + throw new AnalysisException("The field parameter needs to be a foldable string value.") + } + val extractField = field.eval().asInstanceOf[UTF8String].toString.toUpperCase(Locale.ROOT) + extractField match { + case "YEAR" => Year(source) + case "QUARTER" => Quarter(source) + case "MONTH" => Month(source) + case "WEEK" => WeekOfYear(source) + case "DAY" => DayOfMonth(source) + case "DAYOFWEEK" => DayOfWeek(source) + case "DOW" => Subtract(DayOfWeek(source), Literal(1)) + case "ISODOW" => Add(WeekDay(source), Literal(1)) + case "DOY" => DayOfYear(source) + case "HOUR" => Hour(source) + case "MINUTE" => Minute(source) + case "SECOND" => Second(source) + case other => + throw new AnalysisException(s"Literals of type '$other' are currently not supported.") + }}) + } + + override def flatArguments: Iterator[Any] = Iterator(field, source) + override def sql: String = s"$prettyName(${field.sql}, ${source.sql})" + override def prettyName: String = "date_part" +} diff --git a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql new file mode 100644 index 000000000000..40df334d2558 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql @@ -0,0 +1,29 @@ +CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c; + +select date_part('year', c) from t; + +select date_part('quarter', c) from t; + +select date_part('month', c) from t; + +select date_part('week', c) from t; + +select date_part('day', c) from t; + +select date_part('dayofweek', c) from t; + +select date_part('dow', c) from t; + +select date_part('isodow', c) from t; + +select date_part('doy', c) from t; + +select date_part('hour', c) from t; + +select date_part('minute', c) from t; + +select date_part('second', c) from t; + +select date_part('not_supported', c) from t; + +select date_part(c, c) from t; diff --git a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out new file mode 100644 index 000000000000..df6cbca06e4d --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out @@ -0,0 +1,124 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 15 + + +-- !query 0 +CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c +-- !query 0 schema +struct<> +-- !query 0 output + + + +-- !query 1 +select date_part('year', c) from t +-- !query 1 schema +struct +-- !query 1 output +2011 + + +-- !query 2 +select date_part('quarter', c) from t +-- !query 2 schema +struct +-- !query 2 output +2 + + +-- !query 3 +select date_part('month', c) from t +-- !query 3 schema +struct +-- !query 3 output +5 + + +-- !query 4 +select date_part('week', c) from t +-- !query 4 schema +struct +-- !query 4 output +18 + + +-- !query 5 +select date_part('day', c) from t +-- !query 5 schema +struct +-- !query 5 output +6 + + +-- !query 6 +select date_part('dayofweek', c) from t +-- !query 6 schema +struct +-- !query 6 output +6 + + +-- !query 7 +select date_part('dow', c) from t +-- !query 7 schema +struct +-- !query 7 output +5 + + +-- !query 8 +select date_part('isodow', c) from t +-- !query 8 schema +struct +-- !query 8 output +5 + + +-- !query 9 +select date_part('doy', c) from t +-- !query 9 schema +struct +-- !query 9 output +126 + + +-- !query 10 +select date_part('hour', c) from t +-- !query 10 schema +struct +-- !query 10 output +7 + + +-- !query 11 +select date_part('minute', c) from t +-- !query 11 schema +struct +-- !query 11 output +8 + + +-- !query 12 +select date_part('second', c) from t +-- !query 12 schema +struct +-- !query 12 output +9 + + +-- !query 13 +select date_part('not_supported', c) from t +-- !query 13 schema +struct<> +-- !query 13 output +org.apache.spark.sql.AnalysisException +Literals of type 'NOT_SUPPORTED' are currently not supported.;; line 1 pos 7 + + +-- !query 14 +select date_part(c, c) from t +-- !query 14 schema +struct<> +-- !query 14 output +org.apache.spark.sql.AnalysisException +The field parameter needs to be a foldable string value.;; line 1 pos 7 From b856859fca3aacdba7751f5d126315cfa305fa10 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sun, 11 Aug 2019 22:21:03 +0500 Subject: [PATCH 02/24] Support millennium, century and decade by date_part() --- .../expressions/datetimeExpressions.scala | 3 + .../resources/sql-tests/inputs/date_part.sql | 6 + .../sql-tests/results/date_part.sql.out | 108 +++++++++++------- 3 files changed, 75 insertions(+), 42 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index c6e4a0947196..f2b38e2c5674 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1887,6 +1887,9 @@ case class DatePart(field: Expression, source: Expression, child: Expression) } val extractField = field.eval().asInstanceOf[UTF8String].toString.toUpperCase(Locale.ROOT) extractField match { + case "MILLENNIUM" => Millennium(source) + case "CENTURY" => Century(source) + case "DECADE" => Decade(source) case "YEAR" => Year(source) case "QUARTER" => Quarter(source) case "MONTH" => Month(source) diff --git a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql index 40df334d2558..3757ea27f0f4 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql @@ -1,5 +1,11 @@ CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c; +select date_part('millennium', c) from t; + +select date_part('century', c) from t; + +select date_part('decade', c) from t; + select date_part('year', c) from t; select date_part('quarter', c) from t; diff --git a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out index df6cbca06e4d..3b7a6e026f91 100644 --- a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 15 +-- Number of queries: 18 -- !query 0 @@ -11,114 +11,138 @@ struct<> -- !query 1 -select date_part('year', c) from t +select date_part('millennium', c) from t -- !query 1 schema -struct +struct -- !query 1 output -2011 +3 -- !query 2 -select date_part('quarter', c) from t +select date_part('century', c) from t -- !query 2 schema -struct +struct -- !query 2 output -2 +21 -- !query 3 -select date_part('month', c) from t +select date_part('decade', c) from t -- !query 3 schema -struct +struct -- !query 3 output -5 +201 -- !query 4 -select date_part('week', c) from t +select date_part('year', c) from t -- !query 4 schema -struct +struct -- !query 4 output -18 +2011 -- !query 5 -select date_part('day', c) from t +select date_part('quarter', c) from t -- !query 5 schema -struct +struct -- !query 5 output -6 +2 -- !query 6 -select date_part('dayofweek', c) from t +select date_part('month', c) from t -- !query 6 schema -struct +struct -- !query 6 output -6 +5 -- !query 7 -select date_part('dow', c) from t +select date_part('week', c) from t -- !query 7 schema -struct +struct -- !query 7 output -5 +18 -- !query 8 -select date_part('isodow', c) from t +select date_part('day', c) from t -- !query 8 schema -struct +struct -- !query 8 output -5 +6 -- !query 9 -select date_part('doy', c) from t +select date_part('dayofweek', c) from t -- !query 9 schema -struct +struct -- !query 9 output -126 +6 -- !query 10 -select date_part('hour', c) from t +select date_part('dow', c) from t -- !query 10 schema -struct +struct -- !query 10 output -7 +5 -- !query 11 -select date_part('minute', c) from t +select date_part('isodow', c) from t -- !query 11 schema -struct +struct -- !query 11 output -8 +5 -- !query 12 -select date_part('second', c) from t +select date_part('doy', c) from t -- !query 12 schema -struct +struct -- !query 12 output -9 +126 -- !query 13 -select date_part('not_supported', c) from t +select date_part('hour', c) from t -- !query 13 schema -struct<> +struct -- !query 13 output +7 + + +-- !query 14 +select date_part('minute', c) from t +-- !query 14 schema +struct +-- !query 14 output +8 + + +-- !query 15 +select date_part('second', c) from t +-- !query 15 schema +struct +-- !query 15 output +9 + + +-- !query 16 +select date_part('not_supported', c) from t +-- !query 16 schema +struct<> +-- !query 16 output org.apache.spark.sql.AnalysisException Literals of type 'NOT_SUPPORTED' are currently not supported.;; line 1 pos 7 --- !query 14 +-- !query 17 select date_part(c, c) from t --- !query 14 schema +-- !query 17 schema struct<> --- !query 14 output +-- !query 17 output org.apache.spark.sql.AnalysisException The field parameter needs to be a foldable string value.;; line 1 pos 7 From d23b8ba3bde90b1e9b31fd1b8f3970bee58f2de6 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sun, 11 Aug 2019 23:20:22 +0500 Subject: [PATCH 03/24] Uncomment the first usage of date_part() --- .../sql-tests/inputs/pgSQL/timestamp.sql | 11 ++++---- .../sql-tests/results/pgSQL/timestamp.sql.out | 27 ++++++++++++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql index 2b974816766b..9b21057b2934 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql @@ -187,12 +187,11 @@ SELECT '' AS date_trunc_week, date_trunc( 'week', timestamp '2004-02-29 15:44:17 -- WHERE d1 BETWEEN timestamp '1902-01-01' -- AND timestamp '2038-01-01'; --- [SPARK-28420] Date/Time Functions: date_part --- SELECT '' AS "54", d1 as "timestamp", --- date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, --- date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, --- date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second --- FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; +SELECT '' AS `54`, d1 as `timestamp`, + date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, + date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, + date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` + FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; -- SELECT '' AS "54", d1 as "timestamp", -- date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec, diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out index 61fef6eba41a..a744d3d1b590 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 15 +-- Number of queries: 16 -- !query 0 @@ -123,16 +123,31 @@ struct -- !query 13 -SELECT make_timestamp(2014,12,28,6,30,45.887) +SELECT '' AS `54`, d1 as `timestamp`, + date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, + date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, + date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` + FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' -- !query 13 schema -struct +struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute:int,second:int> -- !query 13 output -2014-12-28 06:30:45.887 + 1997-01-02 00:00:00 1997 1 2 0 0 0 + 1997-01-02 03:04:05 1997 1 2 3 4 5 + 1997-02-10 17:32:01 1997 2 10 17 32 1 + 2001-09-22 18:19:20 2001 9 22 18 19 20 -- !query 14 -DROP TABLE TIMESTAMP_TBL +SELECT make_timestamp(2014,12,28,6,30,45.887) -- !query 14 schema -struct<> +struct -- !query 14 output +2014-12-28 06:30:45.887 + + +-- !query 15 +DROP TABLE TIMESTAMP_TBL +-- !query 15 schema +struct<> +-- !query 15 output From af51e524d90253d26dc848d4776328c5f8359d88 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sun, 11 Aug 2019 23:34:04 +0500 Subject: [PATCH 04/24] Reuse date_part() from extract --- .../sql/catalyst/parser/AstBuilder.scala | 35 +------------- .../sql-tests/results/extract.sql.out | 28 +++++------ .../sql-tests/results/pgSQL/date.sql.out | 48 +++++++++---------- 3 files changed, 38 insertions(+), 73 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 7bbdd4f3c520..692399a52d23 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1395,40 +1395,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Create a Extract expression. */ override def visitExtract(ctx: ExtractContext): Expression = withOrigin(ctx) { - ctx.field.getText.toUpperCase(Locale.ROOT) match { - case "MILLENNIUM" => - Millennium(expression(ctx.source)) - case "CENTURY" => - Century(expression(ctx.source)) - case "DECADE" => - Decade(expression(ctx.source)) - case "YEAR" => - Year(expression(ctx.source)) - case "QUARTER" => - Quarter(expression(ctx.source)) - case "MONTH" => - Month(expression(ctx.source)) - case "WEEK" => - WeekOfYear(expression(ctx.source)) - case "DAY" => - DayOfMonth(expression(ctx.source)) - case "DAYOFWEEK" => - DayOfWeek(expression(ctx.source)) - case "DOW" => - Subtract(DayOfWeek(expression(ctx.source)), Literal(1)) - case "ISODOW" => - Add(WeekDay(expression(ctx.source)), Literal(1)) - case "DOY" => - DayOfYear(expression(ctx.source)) - case "HOUR" => - Hour(expression(ctx.source)) - case "MINUTE" => - Minute(expression(ctx.source)) - case "SECOND" => - Second(expression(ctx.source)) - case other => - throw new ParseException(s"Literals of type '$other' are currently not supported.", ctx) - } + new DatePart(Literal(ctx.field.getText), expression(ctx.source)) } /** diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out index 0ca7bdc09b01..617f9bd2aaf1 100644 --- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out @@ -13,7 +13,7 @@ struct<> -- !query 1 select extract(year from c) from t -- !query 1 schema -struct +struct -- !query 1 output 2011 @@ -21,7 +21,7 @@ struct -- !query 2 select extract(quarter from c) from t -- !query 2 schema -struct +struct -- !query 2 output 2 @@ -29,7 +29,7 @@ struct -- !query 3 select extract(month from c) from t -- !query 3 schema -struct +struct -- !query 3 output 5 @@ -37,7 +37,7 @@ struct -- !query 4 select extract(week from c) from t -- !query 4 schema -struct +struct -- !query 4 output 18 @@ -45,7 +45,7 @@ struct -- !query 5 select extract(day from c) from t -- !query 5 schema -struct +struct -- !query 5 output 6 @@ -53,7 +53,7 @@ struct -- !query 6 select extract(dayofweek from c) from t -- !query 6 schema -struct +struct -- !query 6 output 6 @@ -61,7 +61,7 @@ struct -- !query 7 select extract(dow from c) from t -- !query 7 schema -struct<(dayofweek(CAST(c AS DATE)) - 1):int> +struct -- !query 7 output 5 @@ -69,7 +69,7 @@ struct<(dayofweek(CAST(c AS DATE)) - 1):int> -- !query 8 select extract(isodow from c) from t -- !query 8 schema -struct<(weekday(CAST(c AS DATE)) + 1):int> +struct -- !query 8 output 5 @@ -77,7 +77,7 @@ struct<(weekday(CAST(c AS DATE)) + 1):int> -- !query 9 select extract(doy from c) from t -- !query 9 schema -struct +struct -- !query 9 output 126 @@ -85,7 +85,7 @@ struct -- !query 10 select extract(hour from c) from t -- !query 10 schema -struct +struct -- !query 10 output 7 @@ -93,7 +93,7 @@ struct -- !query 11 select extract(minute from c) from t -- !query 11 schema -struct +struct -- !query 11 output 8 @@ -101,7 +101,7 @@ struct -- !query 12 select extract(second from c) from t -- !query 12 schema -struct +struct -- !query 12 output 9 @@ -113,8 +113,6 @@ struct<> -- !query 13 output org.apache.spark.sql.catalyst.parser.ParseException -Literals of type 'NOT_SUPPORTED' are currently not supported.(line 1, pos 7) - +Literals of type 'NOT_SUPPORTED' are currently not supported. == SQL == select extract(not_supported from c) from t --------^^^ diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out index f5586c5a4aa3..00c8e3cd85b6 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out @@ -504,7 +504,7 @@ struct -- !query 47 SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')) -- !query 47 schema -struct +struct -- !query 47 output -2 @@ -512,7 +512,7 @@ struct -- !query 48 SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')) -- !query 48 schema -struct +struct -- !query 48 output -1 @@ -520,7 +520,7 @@ struct -- !query 49 SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')) -- !query 49 schema -struct +struct -- !query 49 output -1 @@ -528,7 +528,7 @@ struct -- !query 50 SELECT EXTRACT(CENTURY FROM DATE '0001-01-01') -- !query 50 schema -struct +struct -- !query 50 output 1 @@ -536,7 +536,7 @@ struct -- !query 51 SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD') -- !query 51 schema -struct +struct -- !query 51 output 1 @@ -544,7 +544,7 @@ struct -- !query 52 SELECT EXTRACT(CENTURY FROM DATE '1900-12-31') -- !query 52 schema -struct +struct -- !query 52 output 19 @@ -552,7 +552,7 @@ struct -- !query 53 SELECT EXTRACT(CENTURY FROM DATE '1901-01-01') -- !query 53 schema -struct +struct -- !query 53 output 20 @@ -560,7 +560,7 @@ struct -- !query 54 SELECT EXTRACT(CENTURY FROM DATE '2000-12-31') -- !query 54 schema -struct +struct -- !query 54 output 20 @@ -568,7 +568,7 @@ struct -- !query 55 SELECT EXTRACT(CENTURY FROM DATE '2001-01-01') -- !query 55 schema -struct +struct -- !query 55 output 21 @@ -584,7 +584,7 @@ true -- !query 57 SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')) -- !query 57 schema -struct +struct -- !query 57 output -1 @@ -592,7 +592,7 @@ struct -- !query 58 SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD') -- !query 58 schema -struct +struct -- !query 58 output 1 @@ -600,7 +600,7 @@ struct -- !query 59 SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31') -- !query 59 schema -struct +struct -- !query 59 output 1 @@ -608,7 +608,7 @@ struct -- !query 60 SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01') -- !query 60 schema -struct +struct -- !query 60 output 2 @@ -616,7 +616,7 @@ struct -- !query 61 SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31') -- !query 61 schema -struct +struct -- !query 61 output 2 @@ -624,7 +624,7 @@ struct -- !query 62 SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01') -- !query 62 schema -struct +struct -- !query 62 output 3 @@ -632,7 +632,7 @@ struct -- !query 63 SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE) -- !query 63 schema -struct +struct -- !query 63 output 3 @@ -640,7 +640,7 @@ struct -- !query 64 SELECT EXTRACT(DECADE FROM DATE '1994-12-25') -- !query 64 schema -struct +struct -- !query 64 output 199 @@ -648,7 +648,7 @@ struct -- !query 65 SELECT EXTRACT(DECADE FROM DATE '0010-01-01') -- !query 65 schema -struct +struct -- !query 65 output 1 @@ -656,7 +656,7 @@ struct -- !query 66 SELECT EXTRACT(DECADE FROM DATE '0009-12-31') -- !query 66 schema -struct +struct -- !query 66 output 0 @@ -664,7 +664,7 @@ struct -- !query 67 SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')) -- !query 67 schema -struct +struct -- !query 67 output 0 @@ -672,7 +672,7 @@ struct -- !query 68 SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')) -- !query 68 schema -struct +struct -- !query 68 output -1 @@ -680,7 +680,7 @@ struct -- !query 69 SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')) -- !query 69 schema -struct +struct -- !query 69 output -1 @@ -688,7 +688,7 @@ struct -- !query 70 SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G')) -- !query 70 schema -struct +struct -- !query 70 output -2 @@ -704,7 +704,7 @@ true -- !query 72 SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000') -- !query 72 schema -struct +struct -- !query 72 output 20 From e68611a5543adfae274549188e0e9945c4c399a3 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Mon, 12 Aug 2019 11:10:59 +0500 Subject: [PATCH 05/24] Add a description for DatePart --- .../expressions/datetimeExpressions.scala | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index f2b38e2c5674..eb1e2f33bc2a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1877,6 +1877,26 @@ case class Decade(child: Expression) extends UnaryExpression with ImplicitCastIn } } +@ExpressionDescription( + usage = "_FUNC_(field, source) - Extracts a part of the date/timestamp.", + arguments = """ + Arguments: + * field - selects which part of the source should be extracted. Supported string values are: + ["MILLENNIUM", "CENTURY", "DECADE", "YEAR", "QUARTER", "MONTH", + "WEEK", "DAY", "DAYOFWEEK", "DOW", "ISODOW", "DOY", + "HOUR", "MINUTE", "SECOND"] + * source - a date (or timestamp) column from where `field` should be extracted + """, + examples = """ + Examples: + > SELECT _FUNC_('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); + 2019 + > SELECT _FUNC_('week', timestamp'2019-08-12 01:00:00.123456'); + 33 + > SELECT _FUNC_('doy', DATE'2019-08-12'); + 224 + """, + since = "3.0.0") case class DatePart(field: Expression, source: Expression, child: Expression) extends RuntimeReplaceable { From ae353b9ffeea3c11d03ff7c4e958389ce106a98d Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 14 Aug 2019 17:46:27 +0500 Subject: [PATCH 06/24] Regen extract.sql.out --- .../sql-tests/results/extract.sql.out | 100 +++++++++--------- 1 file changed, 49 insertions(+), 51 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out index a7efe825c98e..86fbd8e78903 100644 --- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out @@ -13,7 +13,7 @@ struct<> -- !query 1 select extract(millennium from c) from t -- !query 1 schema -struct +struct -- !query 1 output 3 @@ -21,7 +21,7 @@ struct -- !query 2 select extract(millennia from c) from t -- !query 2 schema -struct +struct -- !query 2 output 3 @@ -29,7 +29,7 @@ struct -- !query 3 select extract(mil from c) from t -- !query 3 schema -struct +struct -- !query 3 output 3 @@ -37,7 +37,7 @@ struct -- !query 4 select extract(mils from c) from t -- !query 4 schema -struct +struct -- !query 4 output 3 @@ -45,7 +45,7 @@ struct -- !query 5 select extract(century from c) from t -- !query 5 schema -struct +struct -- !query 5 output 21 @@ -53,7 +53,7 @@ struct -- !query 6 select extract(centuries from c) from t -- !query 6 schema -struct +struct -- !query 6 output 21 @@ -61,7 +61,7 @@ struct -- !query 7 select extract(c from c) from t -- !query 7 schema -struct +struct -- !query 7 output 21 @@ -69,7 +69,7 @@ struct -- !query 8 select extract(cent from c) from t -- !query 8 schema -struct +struct -- !query 8 output 21 @@ -77,7 +77,7 @@ struct -- !query 9 select extract(decade from c) from t -- !query 9 schema -struct +struct -- !query 9 output 201 @@ -85,7 +85,7 @@ struct -- !query 10 select extract(decades from c) from t -- !query 10 schema -struct +struct -- !query 10 output 201 @@ -93,7 +93,7 @@ struct -- !query 11 select extract(dec from c) from t -- !query 11 schema -struct +struct -- !query 11 output 201 @@ -101,7 +101,7 @@ struct -- !query 12 select extract(decs from c) from t -- !query 12 schema -struct +struct -- !query 12 output 201 @@ -109,7 +109,7 @@ struct -- !query 13 select extract(year from c) from t -- !query 13 schema -struct +struct -- !query 13 output 2011 @@ -117,7 +117,7 @@ struct -- !query 14 select extract(y from c) from t -- !query 14 schema -struct +struct -- !query 14 output 2011 @@ -125,7 +125,7 @@ struct -- !query 15 select extract(years from c) from t -- !query 15 schema -struct +struct -- !query 15 output 2011 @@ -133,7 +133,7 @@ struct -- !query 16 select extract(yr from c) from t -- !query 16 schema -struct +struct -- !query 16 output 2011 @@ -141,7 +141,7 @@ struct -- !query 17 select extract(yrs from c) from t -- !query 17 schema -struct +struct -- !query 17 output 2011 @@ -149,7 +149,7 @@ struct -- !query 18 select extract(quarter from c) from t -- !query 18 schema -struct +struct -- !query 18 output 2 @@ -157,7 +157,7 @@ struct -- !query 19 select extract(qtr from c) from t -- !query 19 schema -struct +struct -- !query 19 output 2 @@ -165,7 +165,7 @@ struct -- !query 20 select extract(month from c) from t -- !query 20 schema -struct +struct -- !query 20 output 5 @@ -173,7 +173,7 @@ struct -- !query 21 select extract(mon from c) from t -- !query 21 schema -struct +struct -- !query 21 output 5 @@ -181,7 +181,7 @@ struct -- !query 22 select extract(mons from c) from t -- !query 22 schema -struct +struct -- !query 22 output 5 @@ -189,7 +189,7 @@ struct -- !query 23 select extract(months from c) from t -- !query 23 schema -struct +struct -- !query 23 output 5 @@ -197,7 +197,7 @@ struct -- !query 24 select extract(week from c) from t -- !query 24 schema -struct +struct -- !query 24 output 18 @@ -205,7 +205,7 @@ struct -- !query 25 select extract(w from c) from t -- !query 25 schema -struct +struct -- !query 25 output 18 @@ -213,7 +213,7 @@ struct -- !query 26 select extract(weeks from c) from t -- !query 26 schema -struct +struct -- !query 26 output 18 @@ -221,7 +221,7 @@ struct -- !query 27 select extract(day from c) from t -- !query 27 schema -struct +struct -- !query 27 output 6 @@ -229,7 +229,7 @@ struct -- !query 28 select extract(d from c) from t -- !query 28 schema -struct +struct -- !query 28 output 6 @@ -237,7 +237,7 @@ struct -- !query 29 select extract(days from c) from t -- !query 29 schema -struct +struct -- !query 29 output 6 @@ -245,7 +245,7 @@ struct -- !query 30 select extract(dayofweek from c) from t -- !query 30 schema -struct +struct -- !query 30 output 6 @@ -253,7 +253,7 @@ struct -- !query 31 select extract(dow from c) from t -- !query 31 schema -struct<(dayofweek(CAST(c AS DATE)) - 1):int> +struct -- !query 31 output 5 @@ -261,7 +261,7 @@ struct<(dayofweek(CAST(c AS DATE)) - 1):int> -- !query 32 select extract(isodow from c) from t -- !query 32 schema -struct<(weekday(CAST(c AS DATE)) + 1):int> +struct -- !query 32 output 5 @@ -269,7 +269,7 @@ struct<(weekday(CAST(c AS DATE)) + 1):int> -- !query 33 select extract(doy from c) from t -- !query 33 schema -struct +struct -- !query 33 output 126 @@ -277,7 +277,7 @@ struct -- !query 34 select extract(hour from c) from t -- !query 34 schema -struct +struct -- !query 34 output 7 @@ -285,7 +285,7 @@ struct -- !query 35 select extract(h from c) from t -- !query 35 schema -struct +struct -- !query 35 output 7 @@ -293,7 +293,7 @@ struct -- !query 36 select extract(hours from c) from t -- !query 36 schema -struct +struct -- !query 36 output 7 @@ -301,7 +301,7 @@ struct -- !query 37 select extract(hr from c) from t -- !query 37 schema -struct +struct -- !query 37 output 7 @@ -309,7 +309,7 @@ struct -- !query 38 select extract(hrs from c) from t -- !query 38 schema -struct +struct -- !query 38 output 7 @@ -317,7 +317,7 @@ struct -- !query 39 select extract(minute from c) from t -- !query 39 schema -struct +struct -- !query 39 output 8 @@ -325,7 +325,7 @@ struct -- !query 40 select extract(m from c) from t -- !query 40 schema -struct +struct -- !query 40 output 8 @@ -333,7 +333,7 @@ struct -- !query 41 select extract(min from c) from t -- !query 41 schema -struct +struct -- !query 41 output 8 @@ -341,7 +341,7 @@ struct -- !query 42 select extract(mins from c) from t -- !query 42 schema -struct +struct -- !query 42 output 8 @@ -349,7 +349,7 @@ struct -- !query 43 select extract(minutes from c) from t -- !query 43 schema -struct +struct -- !query 43 output 8 @@ -357,7 +357,7 @@ struct -- !query 44 select extract(second from c) from t -- !query 44 schema -struct +struct -- !query 44 output 9 @@ -365,7 +365,7 @@ struct -- !query 45 select extract(s from c) from t -- !query 45 schema -struct +struct -- !query 45 output 9 @@ -373,7 +373,7 @@ struct -- !query 46 select extract(sec from c) from t -- !query 46 schema -struct +struct -- !query 46 output 9 @@ -381,7 +381,7 @@ struct -- !query 47 select extract(seconds from c) from t -- !query 47 schema -struct +struct -- !query 47 output 9 @@ -389,7 +389,7 @@ struct -- !query 48 select extract(secs from c) from t -- !query 48 schema -struct +struct -- !query 48 output 9 @@ -401,8 +401,6 @@ struct<> -- !query 49 output org.apache.spark.sql.catalyst.parser.ParseException -Literals of type 'NOT_SUPPORTED' are currently not supported.(line 1, pos 7) - +Literals of type 'NOT_SUPPORTED' are currently not supported. == SQL == select extract(not_supported from c) from t --------^^^ From f3b2772afda50f05f61f8aee658e6ea795535006 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 14 Aug 2019 17:49:46 +0500 Subject: [PATCH 07/24] Regen timestamp.sql.out --- .../sql-tests/results/pgSQL/timestamp.sql.out | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out index 13a1d09b71b7..16b13ed93b1b 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 15 +-- Number of queries: 16 -- !query 0 @@ -123,16 +123,31 @@ struct -- !query 13 -SELECT make_timestamp(2014,12,28,6,30,45.887) +SELECT '' AS `54`, d1 as `timestamp`, + date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, + date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, + date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` + FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' -- !query 13 schema -struct +struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute:int,second:int> -- !query 13 output -2014-12-28 06:30:45.887 + 1997-01-02 00:00:00 1997 1 2 0 0 0 + 1997-01-02 03:04:05 1997 1 2 3 4 5 + 1997-02-10 17:32:01 1997 2 10 17 32 1 + 2001-09-22 18:19:20 2001 9 22 18 19 20 -- !query 14 -DROP TABLE TIMESTAMP_TBL +SELECT make_timestamp(2014,12,28,6,30,45.887) -- !query 14 schema -struct<> +struct -- !query 14 output +2014-12-28 06:30:45.887 + + +-- !query 15 +DROP TABLE TIMESTAMP_TBL +-- !query 15 schema +struct<> +-- !query 15 output From b795ebcf7ee071ba600737719062d4f22bcd5c74 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 14 Aug 2019 18:01:55 +0500 Subject: [PATCH 08/24] Add synonyms for field --- .../resources/sql-tests/inputs/date_part.sql | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql index 3757ea27f0f4..cb3d96628100 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql @@ -1,20 +1,41 @@ CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c; select date_part('millennium', c) from t; +select date_part('millennia', c) from t; +select date_part('mil', c) from t; +select date_part('mils', c) from t; select date_part('century', c) from t; +select date_part('centuries', c) from t; +select date_part('c', c) from t; +select date_part('cent', c) from t; select date_part('decade', c) from t; +select date_part('decades', c) from t; +select date_part('dec', c) from t; +select date_part('decs', c) from t; select date_part('year', c) from t; +select date_part('y', c) from t; +select date_part('years', c) from t; +select date_part('yr', c) from t; +select date_part('yrs', c) from t; select date_part('quarter', c) from t; +select date_part('qtr', c) from t; select date_part('month', c) from t; +select date_part('mon', c) from t; +select date_part('mons', c) from t; +select date_part('months', c) from t; select date_part('week', c) from t; +select date_part('w', c) from t; +select date_part('weeks', c) from t; select date_part('day', c) from t; +select date_part('d', c) from t; +select date_part('days', c) from t; select date_part('dayofweek', c) from t; @@ -25,10 +46,22 @@ select date_part('isodow', c) from t; select date_part('doy', c) from t; select date_part('hour', c) from t; +select date_part('h', c) from t; +select date_part('hours', c) from t; +select date_part('hr', c) from t; +select date_part('hrs', c) from t; select date_part('minute', c) from t; +select date_part('m', c) from t; +select date_part('min', c) from t; +select date_part('mins', c) from t; +select date_part('minutes', c) from t; select date_part('second', c) from t; +select date_part('s', c) from t; +select date_part('sec', c) from t; +select date_part('seconds', c) from t; +select date_part('secs', c) from t; select date_part('not_supported', c) from t; From efc3ee010b30e4f5b3c0767760d127f7d6eed674 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 14 Aug 2019 18:02:05 +0500 Subject: [PATCH 09/24] Regen date_part.sql.out --- .../sql-tests/results/date_part.sql.out | 360 +++++++++++++++--- 1 file changed, 312 insertions(+), 48 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out index 3b7a6e026f91..40dce797b56e 100644 --- a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 18 +-- Number of queries: 51 -- !query 0 @@ -19,130 +19,394 @@ struct -- !query 2 -select date_part('century', c) from t +select date_part('millennia', c) from t -- !query 2 schema -struct +struct -- !query 2 output -21 +3 -- !query 3 -select date_part('decade', c) from t +select date_part('mil', c) from t -- !query 3 schema -struct +struct -- !query 3 output -201 +3 -- !query 4 -select date_part('year', c) from t +select date_part('mils', c) from t -- !query 4 schema -struct +struct -- !query 4 output -2011 +3 -- !query 5 -select date_part('quarter', c) from t +select date_part('century', c) from t -- !query 5 schema -struct +struct -- !query 5 output -2 +21 -- !query 6 -select date_part('month', c) from t +select date_part('centuries', c) from t -- !query 6 schema -struct +struct -- !query 6 output -5 +21 -- !query 7 -select date_part('week', c) from t +select date_part('c', c) from t -- !query 7 schema -struct +struct -- !query 7 output -18 +21 -- !query 8 -select date_part('day', c) from t +select date_part('cent', c) from t -- !query 8 schema -struct +struct -- !query 8 output -6 +21 -- !query 9 -select date_part('dayofweek', c) from t +select date_part('decade', c) from t -- !query 9 schema -struct +struct -- !query 9 output -6 +201 -- !query 10 -select date_part('dow', c) from t +select date_part('decades', c) from t -- !query 10 schema -struct +struct -- !query 10 output -5 +201 -- !query 11 -select date_part('isodow', c) from t +select date_part('dec', c) from t -- !query 11 schema -struct +struct -- !query 11 output -5 +201 -- !query 12 -select date_part('doy', c) from t +select date_part('decs', c) from t -- !query 12 schema -struct +struct -- !query 12 output -126 +201 -- !query 13 -select date_part('hour', c) from t +select date_part('year', c) from t -- !query 13 schema -struct +struct -- !query 13 output -7 +2011 -- !query 14 -select date_part('minute', c) from t +select date_part('y', c) from t -- !query 14 schema -struct +struct -- !query 14 output -8 +2011 -- !query 15 -select date_part('second', c) from t +select date_part('years', c) from t -- !query 15 schema -struct +struct -- !query 15 output -9 +2011 -- !query 16 -select date_part('not_supported', c) from t +select date_part('yr', c) from t -- !query 16 schema -struct<> +struct -- !query 16 output +2011 + + +-- !query 17 +select date_part('yrs', c) from t +-- !query 17 schema +struct +-- !query 17 output +2011 + + +-- !query 18 +select date_part('quarter', c) from t +-- !query 18 schema +struct +-- !query 18 output +2 + + +-- !query 19 +select date_part('qtr', c) from t +-- !query 19 schema +struct +-- !query 19 output +2 + + +-- !query 20 +select date_part('month', c) from t +-- !query 20 schema +struct +-- !query 20 output +5 + + +-- !query 21 +select date_part('mon', c) from t +-- !query 21 schema +struct +-- !query 21 output +5 + + +-- !query 22 +select date_part('mons', c) from t +-- !query 22 schema +struct +-- !query 22 output +5 + + +-- !query 23 +select date_part('months', c) from t +-- !query 23 schema +struct +-- !query 23 output +5 + + +-- !query 24 +select date_part('week', c) from t +-- !query 24 schema +struct +-- !query 24 output +18 + + +-- !query 25 +select date_part('w', c) from t +-- !query 25 schema +struct +-- !query 25 output +18 + + +-- !query 26 +select date_part('weeks', c) from t +-- !query 26 schema +struct +-- !query 26 output +18 + + +-- !query 27 +select date_part('day', c) from t +-- !query 27 schema +struct +-- !query 27 output +6 + + +-- !query 28 +select date_part('d', c) from t +-- !query 28 schema +struct +-- !query 28 output +6 + + +-- !query 29 +select date_part('days', c) from t +-- !query 29 schema +struct +-- !query 29 output +6 + + +-- !query 30 +select date_part('dayofweek', c) from t +-- !query 30 schema +struct +-- !query 30 output +6 + + +-- !query 31 +select date_part('dow', c) from t +-- !query 31 schema +struct +-- !query 31 output +5 + + +-- !query 32 +select date_part('isodow', c) from t +-- !query 32 schema +struct +-- !query 32 output +5 + + +-- !query 33 +select date_part('doy', c) from t +-- !query 33 schema +struct +-- !query 33 output +126 + + +-- !query 34 +select date_part('hour', c) from t +-- !query 34 schema +struct +-- !query 34 output +7 + + +-- !query 35 +select date_part('h', c) from t +-- !query 35 schema +struct +-- !query 35 output +7 + + +-- !query 36 +select date_part('hours', c) from t +-- !query 36 schema +struct +-- !query 36 output +7 + + +-- !query 37 +select date_part('hr', c) from t +-- !query 37 schema +struct +-- !query 37 output +7 + + +-- !query 38 +select date_part('hrs', c) from t +-- !query 38 schema +struct +-- !query 38 output +7 + + +-- !query 39 +select date_part('minute', c) from t +-- !query 39 schema +struct +-- !query 39 output +8 + + +-- !query 40 +select date_part('m', c) from t +-- !query 40 schema +struct +-- !query 40 output +8 + + +-- !query 41 +select date_part('min', c) from t +-- !query 41 schema +struct +-- !query 41 output +8 + + +-- !query 42 +select date_part('mins', c) from t +-- !query 42 schema +struct +-- !query 42 output +8 + + +-- !query 43 +select date_part('minutes', c) from t +-- !query 43 schema +struct +-- !query 43 output +8 + + +-- !query 44 +select date_part('second', c) from t +-- !query 44 schema +struct +-- !query 44 output +9 + + +-- !query 45 +select date_part('s', c) from t +-- !query 45 schema +struct +-- !query 45 output +9 + + +-- !query 46 +select date_part('sec', c) from t +-- !query 46 schema +struct +-- !query 46 output +9 + + +-- !query 47 +select date_part('seconds', c) from t +-- !query 47 schema +struct +-- !query 47 output +9 + + +-- !query 48 +select date_part('secs', c) from t +-- !query 48 schema +struct +-- !query 48 output +9 + + +-- !query 49 +select date_part('not_supported', c) from t +-- !query 49 schema +struct<> +-- !query 49 output org.apache.spark.sql.AnalysisException Literals of type 'NOT_SUPPORTED' are currently not supported.;; line 1 pos 7 --- !query 17 +-- !query 50 select date_part(c, c) from t --- !query 17 schema +-- !query 50 schema struct<> --- !query 17 output +-- !query 50 output org.apache.spark.sql.AnalysisException The field parameter needs to be a foldable string value.;; line 1 pos 7 From 9935c01816e09a1be2200e2948c2d6d1c2a49591 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 14 Aug 2019 21:29:16 +0500 Subject: [PATCH 10/24] Add isoyear, milliseconds, microseconds and epoch --- .../sql/catalyst/expressions/datetimeExpressions.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index cc48f1a74f1f..3cfffb0f0dd5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1998,6 +1998,7 @@ case class DatePart(field: Expression, source: Expression, child: Expression) case "CENTURY" | "CENTURIES" | "C" | "CENT" => Century(source) case "DECADE" | "DECADES" | "DEC" | "DECS" => Decade(source) case "YEAR" | "Y" | "YEARS" | "YR" | "YRS" => Year(source) + case "ISOYEAR" => IsoYear(source) case "QUARTER" | "QTR" => Quarter(source) case "MONTH" | "MON" | "MONS" | "MONTHS" => Month(source) case "WEEK" | "W" | "WEEKS" => WeekOfYear(source) @@ -2009,6 +2010,11 @@ case class DatePart(field: Expression, source: Expression, child: Expression) case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => Hour(source) case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => Minute(source) case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" => Second(source) + case "MILLISECONDS" | "MSEC" | "MSECS" | "MILLISECON" | "MSECONDS" | "MS" => + Milliseconds(source) + case "MICROSECONDS" | "USEC" | "USECS" | "USECONDS" | "MICROSECON" | "US" => + Microseconds(source) + case "EPOCH" => Epoch(source) case other => throw new AnalysisException(s"Literals of type '$other' are currently not supported.") }}) From c918eb01da2dbc5b06a78a32864167f89f073ed4 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 14 Aug 2019 21:41:43 +0500 Subject: [PATCH 11/24] Re-gen extract.sql.out --- .../sql-tests/results/extract.sql.out | 128 +++++++++--------- 1 file changed, 63 insertions(+), 65 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out index b02dfe054344..758b5d4afb22 100644 --- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out @@ -13,7 +13,7 @@ struct<> -- !query 1 select extract(millennium from c) from t -- !query 1 schema -struct +struct -- !query 1 output 3 @@ -21,7 +21,7 @@ struct -- !query 2 select extract(millennia from c) from t -- !query 2 schema -struct +struct -- !query 2 output 3 @@ -29,7 +29,7 @@ struct -- !query 3 select extract(mil from c) from t -- !query 3 schema -struct +struct -- !query 3 output 3 @@ -37,7 +37,7 @@ struct -- !query 4 select extract(mils from c) from t -- !query 4 schema -struct +struct -- !query 4 output 3 @@ -45,7 +45,7 @@ struct -- !query 5 select extract(century from c) from t -- !query 5 schema -struct +struct -- !query 5 output 21 @@ -53,7 +53,7 @@ struct -- !query 6 select extract(centuries from c) from t -- !query 6 schema -struct +struct -- !query 6 output 21 @@ -61,7 +61,7 @@ struct -- !query 7 select extract(c from c) from t -- !query 7 schema -struct +struct -- !query 7 output 21 @@ -69,7 +69,7 @@ struct -- !query 8 select extract(cent from c) from t -- !query 8 schema -struct +struct -- !query 8 output 21 @@ -77,7 +77,7 @@ struct -- !query 9 select extract(decade from c) from t -- !query 9 schema -struct +struct -- !query 9 output 201 @@ -85,7 +85,7 @@ struct -- !query 10 select extract(decades from c) from t -- !query 10 schema -struct +struct -- !query 10 output 201 @@ -93,7 +93,7 @@ struct -- !query 11 select extract(dec from c) from t -- !query 11 schema -struct +struct -- !query 11 output 201 @@ -101,7 +101,7 @@ struct -- !query 12 select extract(decs from c) from t -- !query 12 schema -struct +struct -- !query 12 output 201 @@ -109,7 +109,7 @@ struct -- !query 13 select extract(year from c) from t -- !query 13 schema -struct +struct -- !query 13 output 2011 @@ -117,7 +117,7 @@ struct -- !query 14 select extract(y from c) from t -- !query 14 schema -struct +struct -- !query 14 output 2011 @@ -125,7 +125,7 @@ struct -- !query 15 select extract(years from c) from t -- !query 15 schema -struct +struct -- !query 15 output 2011 @@ -133,7 +133,7 @@ struct -- !query 16 select extract(yr from c) from t -- !query 16 schema -struct +struct -- !query 16 output 2011 @@ -141,7 +141,7 @@ struct -- !query 17 select extract(yrs from c) from t -- !query 17 schema -struct +struct -- !query 17 output 2011 @@ -149,7 +149,7 @@ struct -- !query 18 select extract(isoyear from c) from t -- !query 18 schema -struct +struct -- !query 18 output 2011 @@ -157,7 +157,7 @@ struct -- !query 19 select extract(quarter from c) from t -- !query 19 schema -struct +struct -- !query 19 output 2 @@ -165,7 +165,7 @@ struct -- !query 20 select extract(qtr from c) from t -- !query 20 schema -struct +struct -- !query 20 output 2 @@ -173,7 +173,7 @@ struct -- !query 21 select extract(month from c) from t -- !query 21 schema -struct +struct -- !query 21 output 5 @@ -181,7 +181,7 @@ struct -- !query 22 select extract(mon from c) from t -- !query 22 schema -struct +struct -- !query 22 output 5 @@ -189,7 +189,7 @@ struct -- !query 23 select extract(mons from c) from t -- !query 23 schema -struct +struct -- !query 23 output 5 @@ -197,7 +197,7 @@ struct -- !query 24 select extract(months from c) from t -- !query 24 schema -struct +struct -- !query 24 output 5 @@ -205,7 +205,7 @@ struct -- !query 25 select extract(week from c) from t -- !query 25 schema -struct +struct -- !query 25 output 18 @@ -213,7 +213,7 @@ struct -- !query 26 select extract(w from c) from t -- !query 26 schema -struct +struct -- !query 26 output 18 @@ -221,7 +221,7 @@ struct -- !query 27 select extract(weeks from c) from t -- !query 27 schema -struct +struct -- !query 27 output 18 @@ -229,7 +229,7 @@ struct -- !query 28 select extract(day from c) from t -- !query 28 schema -struct +struct -- !query 28 output 6 @@ -237,7 +237,7 @@ struct -- !query 29 select extract(d from c) from t -- !query 29 schema -struct +struct -- !query 29 output 6 @@ -245,7 +245,7 @@ struct -- !query 30 select extract(days from c) from t -- !query 30 schema -struct +struct -- !query 30 output 6 @@ -253,7 +253,7 @@ struct -- !query 31 select extract(dayofweek from c) from t -- !query 31 schema -struct +struct -- !query 31 output 6 @@ -261,7 +261,7 @@ struct -- !query 32 select extract(dow from c) from t -- !query 32 schema -struct<(dayofweek(CAST(c AS DATE)) - 1):int> +struct -- !query 32 output 5 @@ -269,7 +269,7 @@ struct<(dayofweek(CAST(c AS DATE)) - 1):int> -- !query 33 select extract(isodow from c) from t -- !query 33 schema -struct<(weekday(CAST(c AS DATE)) + 1):int> +struct -- !query 33 output 5 @@ -277,7 +277,7 @@ struct<(weekday(CAST(c AS DATE)) + 1):int> -- !query 34 select extract(doy from c) from t -- !query 34 schema -struct +struct -- !query 34 output 126 @@ -285,7 +285,7 @@ struct -- !query 35 select extract(hour from c) from t -- !query 35 schema -struct +struct -- !query 35 output 7 @@ -293,7 +293,7 @@ struct -- !query 36 select extract(h from c) from t -- !query 36 schema -struct +struct -- !query 36 output 7 @@ -301,7 +301,7 @@ struct -- !query 37 select extract(hours from c) from t -- !query 37 schema -struct +struct -- !query 37 output 7 @@ -309,7 +309,7 @@ struct -- !query 38 select extract(hr from c) from t -- !query 38 schema -struct +struct -- !query 38 output 7 @@ -317,7 +317,7 @@ struct -- !query 39 select extract(hrs from c) from t -- !query 39 schema -struct +struct -- !query 39 output 7 @@ -325,7 +325,7 @@ struct -- !query 40 select extract(minute from c) from t -- !query 40 schema -struct +struct -- !query 40 output 8 @@ -333,7 +333,7 @@ struct -- !query 41 select extract(m from c) from t -- !query 41 schema -struct +struct -- !query 41 output 8 @@ -341,7 +341,7 @@ struct -- !query 42 select extract(min from c) from t -- !query 42 schema -struct +struct -- !query 42 output 8 @@ -349,7 +349,7 @@ struct -- !query 43 select extract(mins from c) from t -- !query 43 schema -struct +struct -- !query 43 output 8 @@ -357,7 +357,7 @@ struct -- !query 44 select extract(minutes from c) from t -- !query 44 schema -struct +struct -- !query 44 output 8 @@ -365,7 +365,7 @@ struct -- !query 45 select extract(second from c) from t -- !query 45 schema -struct +struct -- !query 45 output 9 @@ -373,7 +373,7 @@ struct -- !query 46 select extract(s from c) from t -- !query 46 schema -struct +struct -- !query 46 output 9 @@ -381,7 +381,7 @@ struct -- !query 47 select extract(sec from c) from t -- !query 47 schema -struct +struct -- !query 47 output 9 @@ -389,7 +389,7 @@ struct -- !query 48 select extract(seconds from c) from t -- !query 48 schema -struct +struct -- !query 48 output 9 @@ -397,7 +397,7 @@ struct -- !query 49 select extract(secs from c) from t -- !query 49 schema -struct +struct -- !query 49 output 9 @@ -405,7 +405,7 @@ struct -- !query 50 select extract(milliseconds from c) from t -- !query 50 schema -struct +struct -- !query 50 output 9123.456 @@ -413,7 +413,7 @@ struct -- !query 51 select extract(msec from c) from t -- !query 51 schema -struct +struct -- !query 51 output 9123.456 @@ -421,7 +421,7 @@ struct -- !query 52 select extract(msecs from c) from t -- !query 52 schema -struct +struct -- !query 52 output 9123.456 @@ -429,7 +429,7 @@ struct -- !query 53 select extract(millisecon from c) from t -- !query 53 schema -struct +struct -- !query 53 output 9123.456 @@ -437,7 +437,7 @@ struct -- !query 54 select extract(mseconds from c) from t -- !query 54 schema -struct +struct -- !query 54 output 9123.456 @@ -445,7 +445,7 @@ struct -- !query 55 select extract(ms from c) from t -- !query 55 schema -struct +struct -- !query 55 output 9123.456 @@ -453,7 +453,7 @@ struct -- !query 56 select extract(microseconds from c) from t -- !query 56 schema -struct +struct -- !query 56 output 9123456 @@ -461,7 +461,7 @@ struct -- !query 57 select extract(usec from c) from t -- !query 57 schema -struct +struct -- !query 57 output 9123456 @@ -469,7 +469,7 @@ struct -- !query 58 select extract(usecs from c) from t -- !query 58 schema -struct +struct -- !query 58 output 9123456 @@ -477,7 +477,7 @@ struct -- !query 59 select extract(useconds from c) from t -- !query 59 schema -struct +struct -- !query 59 output 9123456 @@ -485,7 +485,7 @@ struct -- !query 60 select extract(microsecon from c) from t -- !query 60 schema -struct +struct -- !query 60 output 9123456 @@ -493,7 +493,7 @@ struct -- !query 61 select extract(us from c) from t -- !query 61 schema -struct +struct -- !query 61 output 9123456 @@ -501,7 +501,7 @@ struct -- !query 62 select extract(epoch from c) from t -- !query 62 schema -struct +struct -- !query 62 output 1304665689.123456 @@ -513,8 +513,6 @@ struct<> -- !query 63 output org.apache.spark.sql.catalyst.parser.ParseException -Literals of type 'NOT_SUPPORTED' are currently not supported.(line 1, pos 7) - +Literals of type 'NOT_SUPPORTED' are currently not supported. == SQL == select extract(not_supported from c) from t --------^^^ From bcf73d22be4c5fd0d48e291cf4466812ace8b55d Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 14 Aug 2019 21:45:41 +0500 Subject: [PATCH 12/24] Re-gen date.sql.out --- .../sql-tests/results/pgSQL/date.sql.out | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out index cb2be6d1cd22..083832007d61 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out @@ -504,7 +504,7 @@ struct -- !query 47 SELECT EXTRACT(EPOCH FROM DATE '1970-01-01') -- !query 47 schema -struct +struct -- !query 47 output 0 @@ -512,7 +512,7 @@ struct -- !query 48 SELECT EXTRACT(EPOCH FROM TIMESTAMP '1970-01-01') -- !query 48 schema -struct +struct -- !query 48 output 0 @@ -520,7 +520,7 @@ struct -- !query 49 SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')) -- !query 49 schema -struct +struct -- !query 49 output -2 @@ -528,7 +528,7 @@ struct -- !query 50 SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')) -- !query 50 schema -struct +struct -- !query 50 output -1 @@ -536,7 +536,7 @@ struct -- !query 51 SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')) -- !query 51 schema -struct +struct -- !query 51 output -1 @@ -544,7 +544,7 @@ struct -- !query 52 SELECT EXTRACT(CENTURY FROM DATE '0001-01-01') -- !query 52 schema -struct +struct -- !query 52 output 1 @@ -552,7 +552,7 @@ struct -- !query 53 SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD') -- !query 53 schema -struct +struct -- !query 53 output 1 @@ -560,7 +560,7 @@ struct -- !query 54 SELECT EXTRACT(CENTURY FROM DATE '1900-12-31') -- !query 54 schema -struct +struct -- !query 54 output 19 @@ -568,7 +568,7 @@ struct -- !query 55 SELECT EXTRACT(CENTURY FROM DATE '1901-01-01') -- !query 55 schema -struct +struct -- !query 55 output 20 @@ -576,7 +576,7 @@ struct -- !query 56 SELECT EXTRACT(CENTURY FROM DATE '2000-12-31') -- !query 56 schema -struct +struct -- !query 56 output 20 @@ -584,7 +584,7 @@ struct -- !query 57 SELECT EXTRACT(CENTURY FROM DATE '2001-01-01') -- !query 57 schema -struct +struct -- !query 57 output 21 @@ -600,7 +600,7 @@ true -- !query 59 SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')) -- !query 59 schema -struct +struct -- !query 59 output -1 @@ -608,7 +608,7 @@ struct -- !query 60 SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD') -- !query 60 schema -struct +struct -- !query 60 output 1 @@ -616,7 +616,7 @@ struct -- !query 61 SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31') -- !query 61 schema -struct +struct -- !query 61 output 1 @@ -624,7 +624,7 @@ struct -- !query 62 SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01') -- !query 62 schema -struct +struct -- !query 62 output 2 @@ -632,7 +632,7 @@ struct -- !query 63 SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31') -- !query 63 schema -struct +struct -- !query 63 output 2 @@ -640,7 +640,7 @@ struct -- !query 64 SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01') -- !query 64 schema -struct +struct -- !query 64 output 3 @@ -648,7 +648,7 @@ struct -- !query 65 SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE) -- !query 65 schema -struct +struct -- !query 65 output 3 @@ -656,7 +656,7 @@ struct -- !query 66 SELECT EXTRACT(DECADE FROM DATE '1994-12-25') -- !query 66 schema -struct +struct -- !query 66 output 199 @@ -664,7 +664,7 @@ struct -- !query 67 SELECT EXTRACT(DECADE FROM DATE '0010-01-01') -- !query 67 schema -struct +struct -- !query 67 output 1 @@ -672,7 +672,7 @@ struct -- !query 68 SELECT EXTRACT(DECADE FROM DATE '0009-12-31') -- !query 68 schema -struct +struct -- !query 68 output 0 @@ -680,7 +680,7 @@ struct -- !query 69 SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')) -- !query 69 schema -struct +struct -- !query 69 output 0 @@ -688,7 +688,7 @@ struct -- !query 70 SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')) -- !query 70 schema -struct +struct -- !query 70 output -1 @@ -696,7 +696,7 @@ struct -- !query 71 SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')) -- !query 71 schema -struct +struct -- !query 71 output -1 @@ -704,7 +704,7 @@ struct -- !query 72 SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G')) -- !query 72 schema -struct +struct -- !query 72 output -2 @@ -720,7 +720,7 @@ true -- !query 74 SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000') -- !query 74 schema -struct +struct -- !query 74 output 20 From 7c549c7a1282740ca2864d439e70a9f181382bba Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Thu, 15 Aug 2019 11:13:40 +0500 Subject: [PATCH 13/24] Uncomment 2 queries in timestamp.sql --- .../sql-tests/inputs/pgSQL/timestamp.sql | 16 ++++---- .../sql-tests/results/pgSQL/timestamp.sql.out | 40 ++++++++++++++++--- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql index 9b21057b2934..cd372f352ddc 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql @@ -193,15 +193,15 @@ SELECT '' AS `54`, d1 as `timestamp`, date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; --- SELECT '' AS "54", d1 as "timestamp", --- date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec, --- date_part( 'usec', d1) AS usec --- FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; +SELECT '' AS `54`, d1 as `timestamp`, + date_part( 'quarter', d1) AS `quarter`, date_part( 'msec', d1) AS `msec`, + date_part( 'usec', d1) AS `usec` + FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; --- SELECT '' AS "54", d1 as "timestamp", --- date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week, --- date_part( 'dow', d1) AS dow --- FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; +SELECT '' AS `54`, d1 as `timestamp`, + date_part( 'isoyear', d1) AS `isoyear`, date_part( 'week', d1) AS `week`, + date_part( 'dow', d1) AS `dow` + FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; -- [SPARK-28137] Data Type Formatting Functions -- TO_CHAR() diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out index 16b13ed93b1b..941350d89024 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 16 +-- Number of queries: 18 -- !query 0 @@ -138,16 +138,44 @@ struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute: -- !query 14 -SELECT make_timestamp(2014,12,28,6,30,45.887) +SELECT '' AS `54`, d1 as `timestamp`, + date_part( 'quarter', d1) AS `quarter`, date_part( 'msec', d1) AS `msec`, + date_part( 'usec', d1) AS `usec` + FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' -- !query 14 schema -struct +struct<54:string,timestamp:timestamp,quarter:int,msec:decimal(8,3),usec:int> -- !query 14 output -2014-12-28 06:30:45.887 + 1997-01-02 00:00:00 1 0 0 + 1997-01-02 03:04:05 1 5000 5000000 + 1997-02-10 17:32:01 1 1000 1000000 + 2001-09-22 18:19:20 3 20000 20000000 -- !query 15 -DROP TABLE TIMESTAMP_TBL +SELECT '' AS `54`, d1 as `timestamp`, + date_part( 'isoyear', d1) AS `isoyear`, date_part( 'week', d1) AS `week`, + date_part( 'dow', d1) AS `dow` + FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' -- !query 15 schema -struct<> +struct<54:string,timestamp:timestamp,isoyear:int,week:int,dow:int> -- !query 15 output + 1997-01-02 00:00:00 1997 1 4 + 1997-01-02 03:04:05 1997 1 4 + 1997-02-10 17:32:01 1997 7 1 + 2001-09-22 18:19:20 2001 38 6 + + +-- !query 16 +SELECT make_timestamp(2014,12,28,6,30,45.887) +-- !query 16 schema +struct +-- !query 16 output +2014-12-28 06:30:45.887 + + +-- !query 17 +DROP TABLE TIMESTAMP_TBL +-- !query 17 schema +struct<> +-- !query 17 output From 1b2c8d4d72394cfd27e8d4e6b0a9291706cd62e5 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Thu, 15 Aug 2019 11:27:01 +0500 Subject: [PATCH 14/24] List all values of field --- .../expressions/datetimeExpressions.scala | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 3cfffb0f0dd5..ebfde4d1cb8a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1969,9 +1969,22 @@ case class Epoch(child: Expression, timeZoneId: Option[String] = None) arguments = """ Arguments: * field - selects which part of the source should be extracted. Supported string values are: - ["MILLENNIUM", "CENTURY", "DECADE", "YEAR", "QUARTER", "MONTH", - "WEEK", "DAY", "DAYOFWEEK", "DOW", "ISODOW", "DOY", - "HOUR", "MINUTE", "SECOND"] + ["MILLENNIUM", "MILLENNIA", "MIL", "MILS", + "CENTURY", "CENTURIES", "C", "CENT", + "DECADE", "DECADES", "DEC", "DECS", + "YEAR", "Y", "YEARS", "YR", "YRS", + "ISOYEAR", + "QUARTER", "QTR", + "MONTH", "MON", "MONS", "MONTHS", + "WEEK", "W", "WEEKS", + "DAY", "D", "DAYS", + "DAYOFWEEK", "DOW", "ISODOW", "DOY", + "HOUR", "H", "HOURS", "HR", "HRS", + "MINUTE", "M", "MIN", "MINS", "MINUTES", + "SECOND", "S", "SEC", "SECONDS", "SECS", + "MILLISECONDS", "MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS", + "MICROSECONDS", "USEC", "USECS", "USECONDS", "MICROSECON", "US", + "EPOCH"] * source - a date (or timestamp) column from where `field` should be extracted """, examples = """ From 2fa25b1d64bd75ee4f56be88f36944f03f623cc5 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Fri, 16 Aug 2019 10:34:04 +0500 Subject: [PATCH 15/24] Put synonyms to () --- .../expressions/datetimeExpressions.scala | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index ebfde4d1cb8a..cc2dae76f30a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1969,21 +1969,24 @@ case class Epoch(child: Expression, timeZoneId: Option[String] = None) arguments = """ Arguments: * field - selects which part of the source should be extracted. Supported string values are: - ["MILLENNIUM", "MILLENNIA", "MIL", "MILS", - "CENTURY", "CENTURIES", "C", "CENT", - "DECADE", "DECADES", "DEC", "DECS", - "YEAR", "Y", "YEARS", "YR", "YRS", + ["MILLENNIUM", ("MILLENNIA", "MIL", "MILS"), + "CENTURY", ("CENTURIES", "C", "CENT"), + "DECADE", ("DECADES", "DEC", "DECS"), + "YEAR", ("Y", "YEARS", "YR", "YRS"), "ISOYEAR", - "QUARTER", "QTR", - "MONTH", "MON", "MONS", "MONTHS", - "WEEK", "W", "WEEKS", - "DAY", "D", "DAYS", - "DAYOFWEEK", "DOW", "ISODOW", "DOY", - "HOUR", "H", "HOURS", "HR", "HRS", - "MINUTE", "M", "MIN", "MINS", "MINUTES", - "SECOND", "S", "SEC", "SECONDS", "SECS", - "MILLISECONDS", "MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS", - "MICROSECONDS", "USEC", "USECS", "USECONDS", "MICROSECON", "US", + "QUARTER", ("QTR"), + "MONTH", ("MON", "MONS", "MONTHS"), + "WEEK", ("W", "WEEKS"), + "DAY", ("D", "DAYS"), + "DAYOFWEEK", + "DOW", + "ISODOW", + "DOY", + "HOUR", ("H", "HOURS", "HR", "HRS"), + "MINUTE", ("M", "MIN", "MINS", "MINUTES"), + "SECOND", ("S", "SEC", "SECONDS", "SECS"), + "MILLISECONDS", ("MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS"), + "MICROSECONDS", ("USEC", "USECS", "USECONDS", "MICROSECON", "US"), "EPOCH"] * source - a date (or timestamp) column from where `field` should be extracted """, From 494679c6523258a80e0c9a08f9b57424b51a2cec Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Fri, 16 Aug 2019 11:31:49 +0500 Subject: [PATCH 16/24] Remove unneeded backquotes --- .../test/resources/sql-tests/inputs/pgSQL/timestamp.sql | 8 ++++---- .../resources/sql-tests/results/pgSQL/timestamp.sql.out | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql index cd372f352ddc..65e8d3280e07 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql @@ -194,13 +194,13 @@ SELECT '' AS `54`, d1 as `timestamp`, FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'quarter', d1) AS `quarter`, date_part( 'msec', d1) AS `msec`, - date_part( 'usec', d1) AS `usec` + date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec, + date_part( 'usec', d1) AS usec FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'isoyear', d1) AS `isoyear`, date_part( 'week', d1) AS `week`, - date_part( 'dow', d1) AS `dow` + date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week, + date_part( 'dow', d1) AS dow FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; -- [SPARK-28137] Data Type Formatting Functions diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out index 941350d89024..75d9ee8d9c79 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out @@ -139,8 +139,8 @@ struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute: -- !query 14 SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'quarter', d1) AS `quarter`, date_part( 'msec', d1) AS `msec`, - date_part( 'usec', d1) AS `usec` + date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec, + date_part( 'usec', d1) AS usec FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' -- !query 14 schema struct<54:string,timestamp:timestamp,quarter:int,msec:decimal(8,3),usec:int> @@ -153,8 +153,8 @@ struct<54:string,timestamp:timestamp,quarter:int,msec:decimal(8,3),usec:int> -- !query 15 SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'isoyear', d1) AS `isoyear`, date_part( 'week', d1) AS `week`, - date_part( 'dow', d1) AS `dow` + date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week, + date_part( 'dow', d1) AS dow FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' -- !query 15 schema struct<54:string,timestamp:timestamp,isoyear:int,week:int,dow:int> From 5c8c34d6f3e747fa468fc99fccdfd7cbf6a16404 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Fri, 16 Aug 2019 11:51:23 +0500 Subject: [PATCH 17/24] Remove backquotes around year, month, day, hour, minute and second --- .../sql-tests/inputs/pgSQL/timestamp.sql | 6 ++--- .../sql-tests/results/pgSQL/timestamp.sql.out | 23 ++++++++++++------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql index 65e8d3280e07..b2a588baf6d4 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql @@ -188,9 +188,9 @@ SELECT '' AS date_trunc_week, date_trunc( 'week', timestamp '2004-02-29 15:44:17 -- AND timestamp '2038-01-01'; SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, - date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, - date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` + date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, + date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, + date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; SELECT '' AS `54`, d1 as `timestamp`, diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out index 75d9ee8d9c79..ff82a2eab5a9 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out @@ -124,17 +124,24 @@ struct -- !query 13 SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, - date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, - date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` + date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, + date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, + date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' -- !query 13 schema -struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute:int,second:int> +struct<> -- !query 13 output - 1997-01-02 00:00:00 1997 1 2 0 0 0 - 1997-01-02 03:04:05 1997 1 2 3 4 5 - 1997-02-10 17:32:01 1997 2 10 17 32 1 - 2001-09-22 18:19:20 2001 9 22 18 19 20 +org.apache.spark.sql.catalyst.parser.ParseException + +no viable alternative at input 'year'(line 2, pos 30) + +== SQL == +SELECT '' AS `54`, d1 as `timestamp`, + date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, +------------------------------^^^ + date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, + date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second + FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' -- !query 14 From 05b37460af8f89649c94b55bbe333e03810ce4ad Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sat, 17 Aug 2019 17:41:59 +0500 Subject: [PATCH 18/24] Revert "Remove backquotes around year, month, day, hour, minute and second" This reverts commit 5c8c34d6f3e747fa468fc99fccdfd7cbf6a16404. --- .../sql-tests/inputs/pgSQL/timestamp.sql | 6 ++--- .../sql-tests/results/pgSQL/timestamp.sql.out | 23 +++++++------------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql index b2a588baf6d4..65e8d3280e07 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql @@ -188,9 +188,9 @@ SELECT '' AS date_trunc_week, date_trunc( 'week', timestamp '2004-02-29 15:44:17 -- AND timestamp '2038-01-01'; SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, - date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, - date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second + date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, + date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, + date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; SELECT '' AS `54`, d1 as `timestamp`, diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out index ff82a2eab5a9..75d9ee8d9c79 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out @@ -124,24 +124,17 @@ struct -- !query 13 SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, - date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, - date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second + date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, + date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, + date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' -- !query 13 schema -struct<> +struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute:int,second:int> -- !query 13 output -org.apache.spark.sql.catalyst.parser.ParseException - -no viable alternative at input 'year'(line 2, pos 30) - -== SQL == -SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, -------------------------------^^^ - date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, - date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second - FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' + 1997-01-02 00:00:00 1997 1 2 0 0 0 + 1997-01-02 03:04:05 1997 1 2 3 4 5 + 1997-02-10 17:32:01 1997 2 10 17 32 1 + 2001-09-22 18:19:20 2001 9 22 18 19 20 -- !query 14 From 74a7fecd82c410be0351998689914ae6b1e55eab Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Fri, 23 Aug 2019 07:56:08 +0300 Subject: [PATCH 19/24] Turn off ansi mode for a query --- .../sql-tests/inputs/pgSQL/timestamp.sql | 11 ++-- .../sql-tests/results/pgSQL/timestamp.sql.out | 52 ++++++++++++------- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql index 65e8d3280e07..becd7b312448 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql @@ -186,13 +186,14 @@ SELECT '' AS date_trunc_week, date_trunc( 'week', timestamp '2004-02-29 15:44:17 -- FROM TIMESTAMP_TBL -- WHERE d1 BETWEEN timestamp '1902-01-01' -- AND timestamp '2038-01-01'; - +-- [SPARK-28767] ParseException: no viable alternative at input 'year' +set spark.sql.parser.ansi.enabled=false; SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, - date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, - date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` + date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, + date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, + date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; - +set spark.sql.parser.ansi.enabled=true; SELECT '' AS `54`, d1 as `timestamp`, date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec, date_part( 'usec', d1) AS usec diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out index 75d9ee8d9c79..88d00c78f5f4 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 18 +-- Number of queries: 20 -- !query 0 @@ -123,59 +123,75 @@ struct -- !query 13 +set spark.sql.parser.ansi.enabled=false +-- !query 13 schema +struct +-- !query 13 output +spark.sql.parser.ansi.enabled false + + +-- !query 14 SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, - date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, - date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` + date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, + date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, + date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' --- !query 13 schema +-- !query 14 schema struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute:int,second:int> --- !query 13 output +-- !query 14 output 1997-01-02 00:00:00 1997 1 2 0 0 0 1997-01-02 03:04:05 1997 1 2 3 4 5 1997-02-10 17:32:01 1997 2 10 17 32 1 2001-09-22 18:19:20 2001 9 22 18 19 20 --- !query 14 +-- !query 15 +set spark.sql.parser.ansi.enabled=true +-- !query 15 schema +struct +-- !query 15 output +spark.sql.parser.ansi.enabled true + + +-- !query 16 SELECT '' AS `54`, d1 as `timestamp`, date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec, date_part( 'usec', d1) AS usec FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' --- !query 14 schema +-- !query 16 schema struct<54:string,timestamp:timestamp,quarter:int,msec:decimal(8,3),usec:int> --- !query 14 output +-- !query 16 output 1997-01-02 00:00:00 1 0 0 1997-01-02 03:04:05 1 5000 5000000 1997-02-10 17:32:01 1 1000 1000000 2001-09-22 18:19:20 3 20000 20000000 --- !query 15 +-- !query 17 SELECT '' AS `54`, d1 as `timestamp`, date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week, date_part( 'dow', d1) AS dow FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' --- !query 15 schema +-- !query 17 schema struct<54:string,timestamp:timestamp,isoyear:int,week:int,dow:int> --- !query 15 output +-- !query 17 output 1997-01-02 00:00:00 1997 1 4 1997-01-02 03:04:05 1997 1 4 1997-02-10 17:32:01 1997 7 1 2001-09-22 18:19:20 2001 38 6 --- !query 16 +-- !query 18 SELECT make_timestamp(2014,12,28,6,30,45.887) --- !query 16 schema +-- !query 18 schema struct --- !query 16 output +-- !query 18 output 2014-12-28 06:30:45.887 --- !query 17 +-- !query 19 DROP TABLE TIMESTAMP_TBL --- !query 17 schema +-- !query 19 schema struct<> --- !query 17 output +-- !query 19 output From bc707dfac3aaccce95a5db2258724469d5ede409 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Fri, 16 Aug 2019 18:57:14 +0900 Subject: [PATCH 20/24] Fix --- .../expressions/datetimeExpressions.scala | 63 +++++++++++-------- .../sql/catalyst/parser/AstBuilder.scala | 7 ++- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index cc2dae76f30a..0280702f30c8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1964,6 +1964,38 @@ case class Epoch(child: Expression, timeZoneId: Option[String] = None) } } +object DatePart { + + def parseExtractField( + extractField: String, + source: Expression, + errorHandleFunc: => Unit): Expression = extractField.toUpperCase(Locale.ROOT) match { + case "MILLENNIUM" | "MILLENNIA" | "MIL" | "MILS" => Millennium(source) + case "CENTURY" | "CENTURIES" | "C" | "CENT" => Century(source) + case "DECADE" | "DECADES" | "DEC" | "DECS" => Decade(source) + case "YEAR" | "Y" | "YEARS" | "YR" | "YRS" => Year(source) + case "ISOYEAR" => IsoYear(source) + case "QUARTER" | "QTR" => Quarter(source) + case "MONTH" | "MON" | "MONS" | "MONTHS" => Month(source) + case "WEEK" | "W" | "WEEKS" => WeekOfYear(source) + case "DAY" | "D" | "DAYS" => DayOfMonth(source) + case "DAYOFWEEK" => DayOfWeek(source) + case "DOW" => Subtract(DayOfWeek(source), Literal(1)) + case "ISODOW" => Add(WeekDay(source), Literal(1)) + case "DOY" => DayOfYear(source) + case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => Hour(source) + case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => Minute(source) + case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" => Second(source) + case "MILLISECONDS" | "MSEC" | "MSECS" | "MILLISECON" | "MSECONDS" | "MS" => + Milliseconds(source) + case "MICROSECONDS" | "USEC" | "USECS" | "USECONDS" | "MICROSECON" | "US" => + Microseconds(source) + case "EPOCH" => Epoch(source) + case other => + errorHandleFunc.asInstanceOf[Expression] + } +} + @ExpressionDescription( usage = "_FUNC_(field, source) - Extracts a part of the date/timestamp.", arguments = """ @@ -2008,32 +2040,11 @@ case class DatePart(field: Expression, source: Expression, child: Expression) if (!field.foldable) { throw new AnalysisException("The field parameter needs to be a foldable string value.") } - val extractField = field.eval().asInstanceOf[UTF8String].toString.toUpperCase(Locale.ROOT) - extractField match { - case "MILLENNIUM" | "MILLENNIA" | "MIL" | "MILS" => Millennium(source) - case "CENTURY" | "CENTURIES" | "C" | "CENT" => Century(source) - case "DECADE" | "DECADES" | "DEC" | "DECS" => Decade(source) - case "YEAR" | "Y" | "YEARS" | "YR" | "YRS" => Year(source) - case "ISOYEAR" => IsoYear(source) - case "QUARTER" | "QTR" => Quarter(source) - case "MONTH" | "MON" | "MONS" | "MONTHS" => Month(source) - case "WEEK" | "W" | "WEEKS" => WeekOfYear(source) - case "DAY" | "D" | "DAYS" => DayOfMonth(source) - case "DAYOFWEEK" => DayOfWeek(source) - case "DOW" => Subtract(DayOfWeek(source), Literal(1)) - case "ISODOW" => Add(WeekDay(source), Literal(1)) - case "DOY" => DayOfYear(source) - case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => Hour(source) - case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => Minute(source) - case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" => Second(source) - case "MILLISECONDS" | "MSEC" | "MSECS" | "MILLISECON" | "MSECONDS" | "MS" => - Milliseconds(source) - case "MICROSECONDS" | "USEC" | "USECS" | "USECONDS" | "MICROSECON" | "US" => - Microseconds(source) - case "EPOCH" => Epoch(source) - case other => - throw new AnalysisException(s"Literals of type '$other' are currently not supported.") - }}) + val fieldStr = field.eval().asInstanceOf[UTF8String].toString + DatePart.parseExtractField(fieldStr, source, { + throw new AnalysisException(s"Literals of type '$fieldStr' are currently not supported.") + }) + }) } override def flatArguments: Iterator[Any] = Iterator(field, source) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 232136dfe47c..b2a0c4727772 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1409,7 +1409,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Create a Extract expression. */ override def visitExtract(ctx: ExtractContext): Expression = withOrigin(ctx) { - new DatePart(Literal(ctx.field.getText), expression(ctx.source)) + val fieldStr = ctx.field.getText + val source = expression(ctx.source) + val extractField = DatePart.parseExtractField(fieldStr, source, { + throw new ParseException(s"Literals of type '$fieldStr' are currently not supported.", ctx) + }) + new DatePart(Literal(fieldStr), expression(ctx.source), extractField) } /** From d86c092890a4342bef593cd262ad333036689267 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Fri, 23 Aug 2019 22:31:07 +0300 Subject: [PATCH 21/24] Re-gen date_part.sql.out --- sql/core/src/test/resources/sql-tests/results/date_part.sql.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out index 40dce797b56e..c59dfdbd3da3 100644 --- a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out @@ -400,7 +400,7 @@ select date_part('not_supported', c) from t struct<> -- !query 49 output org.apache.spark.sql.AnalysisException -Literals of type 'NOT_SUPPORTED' are currently not supported.;; line 1 pos 7 +Literals of type 'not_supported' are currently not supported.;; line 1 pos 7 -- !query 50 From ade541d4bbdc0143486b77be75176b79c1dc8c41 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Fri, 23 Aug 2019 22:35:31 +0300 Subject: [PATCH 22/24] Re-gen extract.sql.out --- sql/core/src/test/resources/sql-tests/results/extract.sql.out | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out index 758b5d4afb22..e007fa8368ef 100644 --- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out @@ -513,6 +513,8 @@ struct<> -- !query 63 output org.apache.spark.sql.catalyst.parser.ParseException -Literals of type 'NOT_SUPPORTED' are currently not supported. +Literals of type 'not_supported' are currently not supported.(line 1, pos 7) + == SQL == select extract(not_supported from c) from t +-------^^^ From 43968c271f734d60cc13238e42e49c833c620033 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 4 Sep 2019 20:45:58 +0500 Subject: [PATCH 23/24] Change type of errorHandleFunc to Nothing --- .../spark/sql/catalyst/expressions/datetimeExpressions.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 0280702f30c8..9d43701f0305 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1969,7 +1969,7 @@ object DatePart { def parseExtractField( extractField: String, source: Expression, - errorHandleFunc: => Unit): Expression = extractField.toUpperCase(Locale.ROOT) match { + errorHandleFunc: => Nothing): Expression = extractField.toUpperCase(Locale.ROOT) match { case "MILLENNIUM" | "MILLENNIA" | "MIL" | "MILS" => Millennium(source) case "CENTURY" | "CENTURIES" | "C" | "CENT" => Century(source) case "DECADE" | "DECADES" | "DEC" | "DECS" => Decade(source) @@ -1991,8 +1991,7 @@ object DatePart { case "MICROSECONDS" | "USEC" | "USECS" | "USECONDS" | "MICROSECON" | "US" => Microseconds(source) case "EPOCH" => Epoch(source) - case other => - errorHandleFunc.asInstanceOf[Expression] + case _ => errorHandleFunc } } From 600eee6d8ceef00f6554b64d03aaff404917c975 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Fri, 6 Sep 2019 11:55:35 +0500 Subject: [PATCH 24/24] Use backquotes around year, month, day, hour, minute and second --- .../sql-tests/inputs/pgSQL/timestamp.sql | 11 ++-- .../sql-tests/results/pgSQL/timestamp.sql.out | 52 +++++++------------ 2 files changed, 23 insertions(+), 40 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql index becd7b312448..65e8d3280e07 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql @@ -186,14 +186,13 @@ SELECT '' AS date_trunc_week, date_trunc( 'week', timestamp '2004-02-29 15:44:17 -- FROM TIMESTAMP_TBL -- WHERE d1 BETWEEN timestamp '1902-01-01' -- AND timestamp '2038-01-01'; --- [SPARK-28767] ParseException: no viable alternative at input 'year' -set spark.sql.parser.ansi.enabled=false; + SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, - date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, - date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second + date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, + date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, + date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'; -set spark.sql.parser.ansi.enabled=true; + SELECT '' AS `54`, d1 as `timestamp`, date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec, date_part( 'usec', d1) AS usec diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out index 88d00c78f5f4..75d9ee8d9c79 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 20 +-- Number of queries: 18 -- !query 0 @@ -123,75 +123,59 @@ struct -- !query 13 -set spark.sql.parser.ansi.enabled=false --- !query 13 schema -struct --- !query 13 output -spark.sql.parser.ansi.enabled false - - --- !query 14 SELECT '' AS `54`, d1 as `timestamp`, - date_part( 'year', d1) AS year, date_part( 'month', d1) AS month, - date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour, - date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second + date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, + date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, + date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' --- !query 14 schema +-- !query 13 schema struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute:int,second:int> --- !query 14 output +-- !query 13 output 1997-01-02 00:00:00 1997 1 2 0 0 0 1997-01-02 03:04:05 1997 1 2 3 4 5 1997-02-10 17:32:01 1997 2 10 17 32 1 2001-09-22 18:19:20 2001 9 22 18 19 20 --- !query 15 -set spark.sql.parser.ansi.enabled=true --- !query 15 schema -struct --- !query 15 output -spark.sql.parser.ansi.enabled true - - --- !query 16 +-- !query 14 SELECT '' AS `54`, d1 as `timestamp`, date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec, date_part( 'usec', d1) AS usec FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' --- !query 16 schema +-- !query 14 schema struct<54:string,timestamp:timestamp,quarter:int,msec:decimal(8,3),usec:int> --- !query 16 output +-- !query 14 output 1997-01-02 00:00:00 1 0 0 1997-01-02 03:04:05 1 5000 5000000 1997-02-10 17:32:01 1 1000 1000000 2001-09-22 18:19:20 3 20000 20000000 --- !query 17 +-- !query 15 SELECT '' AS `54`, d1 as `timestamp`, date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week, date_part( 'dow', d1) AS dow FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' --- !query 17 schema +-- !query 15 schema struct<54:string,timestamp:timestamp,isoyear:int,week:int,dow:int> --- !query 17 output +-- !query 15 output 1997-01-02 00:00:00 1997 1 4 1997-01-02 03:04:05 1997 1 4 1997-02-10 17:32:01 1997 7 1 2001-09-22 18:19:20 2001 38 6 --- !query 18 +-- !query 16 SELECT make_timestamp(2014,12,28,6,30,45.887) --- !query 18 schema +-- !query 16 schema struct --- !query 18 output +-- !query 16 output 2014-12-28 06:30:45.887 --- !query 19 +-- !query 17 DROP TABLE TIMESTAMP_TBL --- !query 19 schema +-- !query 17 schema struct<> --- !query 19 output +-- !query 17 output