Skip to content

[SPARK-30301][SQL] Fix wrong results when datetimes as fields of complex types #26942

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
yaooqinn wants to merge 8 commits into from
Closed

[SPARK-30301][SQL] Fix wrong results when datetimes as fields of complex types #26942

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
87fc4ab
[SPARK-30301][SQL] Datetimes as fields of complex types to hive strin…
yaooqinn Dec 19, 2019
7e1c437
unify
yaooqinn Dec 19, 2019
5b8bec3
nit
yaooqinn Dec 19, 2019
bb77e3c
fix tests
yaooqinn Dec 19, 2019
1d4a9bf
fix tests
yaooqinn Dec 19, 2019
e4bcc8c
fix hive compatibilty
yaooqinn Dec 19, 2019
32383e6
regen all g files
yaooqinn Dec 19, 2019
8310be0
fix examples
yaooqinn Dec 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import org.apache.spark.unsafe.types.UTF8String
> SELECT _FUNC_('1, 0.8', 'a INT, b DOUBLE');
{"a":1,"b":0.8}
> SELECT _FUNC_('26/08/2015', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'));
{"time":2015-08-26 00:00:00.0}
{"time":2015-08-26 00:00:00}
""",
since = "3.0.0")
// scalastyle:on line.size.limit
Expand Down
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -505,7 +505,7 @@ case class JsonTuple(children: Seq[Expression])
> SELECT _FUNC_('{"a":1, "b":0.8}', 'a INT, b DOUBLE');
{"a":1,"b":0.8}
> SELECT _FUNC_('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'));
{"time":2015-08-26 00:00:00.0}
{"time":2015-08-26 00:00:00}
""",
since = "2.2.0")
// scalastyle:on line.size.limit
Expand Down
75 changes: 19 additions & 56 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,78 +56,41 @@ object HiveResult {
// We need the types so we can output struct field names
val types = executedPlan.output.map(_.dataType)
// Reformat to match hive tab delimited output.
result.map(_.zip(types).map(toHiveString)).map(_.mkString("\t"))
result.map(_.zip(types).map(e => toHiveString(e)))
.map(_.mkString("\t"))
}

private val primitiveTypes = Seq(
StringType,
IntegerType,
LongType,
DoubleType,
FloatType,
BooleanType,
ByteType,
ShortType,
DateType,
TimestampType,
BinaryType)

private lazy val zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)
private lazy val dateFormatter = DateFormatter(zoneId)
private lazy val timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId)

/** Hive outputs fields of structs slightly differently than top level attributes. */
private def toHiveStructString(a: (Any, DataType)): String = a match {
case (struct: Row, StructType(fields)) =>
struct.toSeq.zip(fields).map {
case (v, t) => s""""${t.name}":${toHiveStructString((v, t.dataType))}"""
}.mkString("{", ",", "}")
case (seq: Seq[_], ArrayType(typ, _)) =>
seq.map(v => (v, typ)).map(toHiveStructString).mkString("[", ",", "]")
case (map: Map[_, _], MapType(kType, vType, _)) =>
map.map {
case (key, value) =>
toHiveStructString((key, kType)) + ":" + toHiveStructString((value, vType))
}.toSeq.sorted.mkString("{", ",", "}")
case (null, _) => "null"
case (s: String, StringType) => "\"" + s + "\""
case (decimal, DecimalType()) => decimal.toString
case (interval: CalendarInterval, CalendarIntervalType) =>
SQLConf.get.intervalOutputStyle match {
case SQL_STANDARD => toSqlStandardString(interval)
case ISO_8601 => toIso8601String(interval)
case MULTI_UNITS => toMultiUnitsString(interval)
}
case (other, tpe) if primitiveTypes contains tpe => other.toString
Copy link
Contributor

@juliuszsompolski juliuszsompolski Mar 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@yaooqinn @cloud-fan
for ArrayType(ByteType), this was returning something like "[B@ba4f370" (by calling toString on the Array[Byte]), after this change it returns the String created via new String(bin, StandardCharsets.UTF_8)
Which of these is the intended output?
[B@ba4f370 is useless, while the string made from binary at least contains the contents of it, so it seems that after this change it at least makes more sense...

}

/** Formats a datum (based on the given data type) and returns the string representation. */
def toHiveString(a: (Any, DataType)): String = a match {
case (struct: Row, StructType(fields)) =>
struct.toSeq.zip(fields).map {
case (v, t) => s""""${t.name}":${toHiveStructString((v, t.dataType))}"""
}.mkString("{", ",", "}")
case (seq: Seq[_], ArrayType(typ, _)) =>
seq.map(v => (v, typ)).map(toHiveStructString).mkString("[", ",", "]")
case (map: Map[_, _], MapType(kType, vType, _)) =>
map.map {
case (key, value) =>
toHiveStructString((key, kType)) + ":" + toHiveStructString((value, vType))
}.toSeq.sorted.mkString("{", ",", "}")
case (null, _) => "NULL"
def toHiveString(a: (Any, DataType), nested: Boolean = false): String = a match {
case (null, _) => if (nested) "null" else "NULL"
Copy link
Contributor

@cloud-fan cloud-fan Dec 20, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this weird behavior inherited from Hive?

Copy link
Member Author

@yaooqinn yaooqinn Dec 20, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, there are many hive compatibility unit tests there

case (b, BooleanType) => b.toString
case (d: Date, DateType) => dateFormatter.format(DateTimeUtils.fromJavaDate(d))
case (t: Timestamp, TimestampType) =>
DateTimeUtils.timestampToString(timestampFormatter, DateTimeUtils.fromJavaTimestamp(t))
timestampFormatter.format(DateTimeUtils.fromJavaTimestamp(t))
case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8)
Copy link
Contributor

@juliuszsompolski juliuszsompolski Mar 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On the other hand, if the nested-struct result is to be parsed back e.g. as json, the arbitrary bytes from the String created from binary may mess it up...
Maybe returning if (nested) "<BINARY_BLOB>" else new String(bin, StandardCharsets.UTF_8) would make sense here?

Copy link
Contributor

@cloud-fan cloud-fan Mar 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a good point. Maybe we can follow df.show

case binary: Array[Byte] => binary.map("%02X".format(_)).mkString("[", " ", "]")

The binary will be displayed like [a0b87...], which will not conflict with json strings.

Copy link
Member

@HyukjinKwon HyukjinKwon Mar 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm, but IIRC Hive results string from binary, and this is toHiveString. Does Hive return <BINARY_BLOB>?

Copy link
Member

@HyukjinKwon HyukjinKwon Mar 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It reminds me of #25480 and #25379

Copy link
Member Author

@yaooqinn yaooqinn Mar 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

hive> create table bt(k binary);
OK
Time taken: 0.637 seconds
hive> insert into bt values ("spark"), ("hello"), ("3"), ("."), ("0");
Query ID = root_20200325055555_1c227d63-47dd-4899-a879-0b6a98269908
Total jobs = 3
Launching Job 1 out of 3
Number of reduce tasks is set to 0 since there's no reduce operator
Starting Job = job_1585115622286_0001, Tracking URL = http://quickstart.cloudera:8088/proxy/application_1585115622286_0001/
Kill Command = /usr/lib/hadoop/bin/hadoop job  -kill job_1585115622286_0001
Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0
2020-03-25 05:57:03,557 Stage-1 map = 0%,  reduce = 0%
2020-03-25 05:57:09,766 Stage-1 map = 100%,  reduce = 0%, Cumulative CPU 1.86 sec
MapReduce Total cumulative CPU time: 1 seconds 860 msec
Ended Job = job_1585115622286_0001
Stage-4 is selected by condition resolver.
Stage-3 is filtered out by condition resolver.
Stage-5 is filtered out by condition resolver.
Moving data to: hdfs://quickstart.cloudera:8020/user/hive/warehouse/bt/.hive-staging_hive_2020-03-25_05-56-54_007_226069574359185383-1/-ext-10000
Loading data to table default.bt
Table default.bt stats: [numFiles=1, numRows=5, totalSize=33, rawDataSize=28]
MapReduce Jobs Launched:
Stage-Stage-1: Map: 1   Cumulative CPU: 1.86 sec   HDFS Read: 3080 HDFS Write: 99 SUCCESS
Total MapReduce CPU Time Spent: 1 seconds 860 msec
OK
Time taken: 17.185 seconds
hive> select k, array(k) from bt;
OK
spark	[spark]
hello	[hello]
3	[3]
.	[.]
0	[0]

Copy link
Member Author

@yaooqinn yaooqinn Mar 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Anything I am missing here?

Copy link
Member

@HyukjinKwon HyukjinKwon Mar 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

array(k) seems the array type of byte

Copy link
Contributor

@cloud-fan cloud-fan Mar 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah i see. Then seems nothing needs to be changed. cc @juliuszsompolski

Copy link
Member Author

@yaooqinn yaooqinn Mar 25, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

0: jdbc:hive2://localhost:10000> desc bt_arr;
INFO  : Compiling command(queryId=hive_20200325062323_10f71e38-fe8d-4094-aa87-5361df066edb): desc bt_arr
INFO  : Semantic Analysis Completed
INFO  : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:col_name, type:string, comment:from deserializer), FieldSchema(name:data_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null)
INFO  : Completed compiling command(queryId=hive_20200325062323_10f71e38-fe8d-4094-aa87-5361df066edb); Time taken: 0.056 seconds
INFO  : Concurrency mode is disabled, not creating a lock manager
INFO  : Executing command(queryId=hive_20200325062323_10f71e38-fe8d-4094-aa87-5361df066edb): desc bt_arr
INFO  : Starting task [Stage-0:DDL] in serial mode
INFO  : Completed executing command(queryId=hive_20200325062323_10f71e38-fe8d-4094-aa87-5361df066edb); Time taken: 0.02 seconds
INFO  : OK
+-----------+----------------+----------+--+
| col_name  |   data_type    | comment  |
+-----------+----------------+----------+--+
| _c0       | array<binary>  |          |
+-----------+----------------+----------+--+
1 row selected (0.158 seconds)
0: jdbc:hive2://localhost:10000> select * from bt_arr;
INFO  : Compiling command(queryId=hive_20200325062323_edfa2e37-a8da-481b-97e9-f4ed9a37b9a4): select * from bt_arr
INFO  : Semantic Analysis Completed
INFO  : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:bt_arr._c0, type:array<binary>, comment:null)], properties:null)
INFO  : Completed compiling command(queryId=hive_20200325062323_edfa2e37-a8da-481b-97e9-f4ed9a37b9a4); Time taken: 0.053 seconds
INFO  : Concurrency mode is disabled, not creating a lock manager
INFO  : Executing command(queryId=hive_20200325062323_edfa2e37-a8da-481b-97e9-f4ed9a37b9a4): select * from bt_arr
INFO  : Completed executing command(queryId=hive_20200325062323_edfa2e37-a8da-481b-97e9-f4ed9a37b9a4); Time taken: 0.001 seconds
INFO  : OK
+-------------+--+
| bt_arr._c0  |
+-------------+--+
| [spark]     |
| [hello]     |
| [3]         |
| [.]         |
| [0]         |
+-------------+--+

Copy link
Contributor

@juliuszsompolski juliuszsompolski Mar 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe the new behaviour is more sensible than the previous - returning just [B@ba4f370 was useless; returning the String made from Binary blob at least returns some content... I know some systems try to parse back these results as JSON to be able to explore the nested data - e.g. I think PowerBI does that with what thriftserver returns... I fear that if the binary has some strange contents, and is unquoted, it will break that JSON parsing... But I haven't tested it.
I don't expect anyone using this kind of type combination and running into issues in practice...

case (decimal: java.math.BigDecimal, DecimalType()) => decimal.toPlainString
case (n, _: NumericType) => n.toString
case (s: String, StringType) => if (nested) "\"" + s + "\"" else s
case (interval: CalendarInterval, CalendarIntervalType) =>
SQLConf.get.intervalOutputStyle match {
Copy link
Contributor

@cloud-fan cloud-fan Dec 19, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not related to this PR, but we should also simplify it. We can output SQL standard format if ansi mode is enabled, and output multi-unit format otherwise.

Copy link
Contributor

@cloud-fan cloud-fan Dec 19, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cc @gengliangwang @xuanyuanking

Copy link
Member Author

@yaooqinn yaooqinn Dec 19, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we are doing so, I suggest we support SQL standard input first

case SQL_STANDARD => toSqlStandardString(interval)
case ISO_8601 => toIso8601String(interval)
case MULTI_UNITS => toMultiUnitsString(interval)
}
case (interval, CalendarIntervalType) => interval.toString
case (other, _ : UserDefinedType[_]) => other.toString
case (other, tpe) if primitiveTypes.contains(tpe) => other.toString
case (seq: Seq[_], ArrayType(typ, _)) =>
seq.map(v => (v, typ)).map(e => toHiveString(e, true)).mkString("[", ",", "]")
case (m: Map[_, _], MapType(kType, vType, _)) =>
m.map { case (key, value) =>
toHiveString((key, kType), true) + ":" + toHiveString((value, vType), true)
}.toSeq.sorted.mkString("{", ",", "}")
case (struct: Row, StructType(fields)) =>
struct.toSeq.zip(fields).map { case (v, t) =>
s""""${t.name}":${toHiveString((v, t.dataType), true)}"""
}.mkString("{", ",", "}")
case (other, _: UserDefinedType[_]) => other.toString
}
}
4 changes: 2 additions & 2 deletions sql/core/src/test/resources/sql-tests/results/array.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ select * from primitive_arrays
-- !query 5 schema
struct<boolean_array:array<boolean>,tinyint_array:array<tinyint>,smallint_array:array<smallint>,int_array:array<int>,bigint_array:array<bigint>,decimal_array:array<decimal(19,0)>,double_array:array<double>,float_array:array<float>,date_array:array<date>,timestamp_array:array<timestamp>>
-- !query 5 output
[true] [2,1] [2,1] [2,1] [2,1] [9223372036854775809,9223372036854775808] [2.0,1.0] [2.0,1.0] [2016-03-14,2016-03-13] [2016-11-15 20:54:00.0,2016-11-12 20:54:00.0]
[true] [2,1] [2,1] [2,1] [2,1] [9223372036854775809,9223372036854775808] [2.0,1.0] [2.0,1.0] [2016-03-14,2016-03-13] [2016-11-15 20:54:00,2016-11-12 20:54:00]


-- !query 6
Expand Down Expand Up @@ -122,7 +122,7 @@ from primitive_arrays
-- !query 8 schema
struct<sort_array(boolean_array, true):array<boolean>,sort_array(tinyint_array, true):array<tinyint>,sort_array(smallint_array, true):array<smallint>,sort_array(int_array, true):array<int>,sort_array(bigint_array, true):array<bigint>,sort_array(decimal_array, true):array<decimal(19,0)>,sort_array(double_array, true):array<double>,sort_array(float_array, true):array<float>,sort_array(date_array, true):array<date>,sort_array(timestamp_array, true):array<timestamp>>
-- !query 8 output
[true] [1,2] [1,2] [1,2] [1,2] [9223372036854775808,9223372036854775809] [1.0,2.0] [1.0,2.0] [2016-03-13,2016-03-14] [2016-11-12 20:54:00.0,2016-11-15 20:54:00.0]
[true] [1,2] [1,2] [1,2] [1,2] [9223372036854775808,9223372036854775809] [1.0,2.0] [1.0,2.0] [2016-03-13,2016-03-14] [2016-11-12 20:54:00,2016-11-15 20:54:00]


-- !query 9
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ select from_csv('26/08/2015', 'time Timestamp', map('timestampFormat', 'dd/MM/yy
-- !query 1 schema
struct<from_csv(26/08/2015):struct<time:timestamp>>
-- !query 1 output
{"time":2015-08-26 00:00:00.0}
{"time":2015-08-26 00:00:00}


-- !query 2
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/resources/sql-tests/results/inline-table.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,4 +150,4 @@ select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-
-- !query 16 schema
struct<a:timestamp,b:array<timestamp>>
-- !query 16 output
1991-12-06 00:00:00 [1991-12-06 01:00:00.0,1991-12-06 12:00:00.0]
1991-12-06 00:00:00 [1991-12-06 01:00:00,1991-12-06 12:00:00]
2 changes: 1 addition & 1 deletion sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ select from_json('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat
-- !query 12 schema
struct<from_json({"time":"26/08/2015"}):struct<time:timestamp>>
-- !query 12 output
{"time":2015-08-26 00:00:00.0}
{"time":2015-08-26 00:00:00}


-- !query 13
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/resources/sql-tests/results/typeCoercion/native/concat.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ FROM various_arrays
-- !query 12 schema
struct<boolean_array:array<boolean>,tinyint_array:array<tinyint>,smallint_array:array<smallint>,int_array:array<int>,bigint_array:array<bigint>,decimal_array:array<decimal(19,0)>,double_array:array<double>,float_array:array<float>,data_array:array<date>,timestamp_array:array<timestamp>,string_array:array<string>,array_array:array<array<string>>,struct_array:array<struct<col1:string,col2:int>>,map_array:array<map<string,int>>>
-- !query 12 output
[true,false,true] [2,1,3,4] [2,1,3,4] [2,1,3,4] [2,1,3,4] [9223372036854775809,9223372036854775808,9223372036854775808,9223372036854775809] [2.0,1.0,3.0,4.0] [2.0,1.0,3.0,4.0] [2016-03-14,2016-03-13,2016-03-12,2016-03-11] [2016-11-15 20:54:00.0,2016-11-12 20:54:00.0,2016-11-11 20:54:00.0] ["a","b","c","d"] [["a","b"],["c","d"],["e"],["f"]] [{"col1":"a","col2":1},{"col1":"b","col2":2},{"col1":"c","col2":3},{"col1":"d","col2":4}] [{"a":1},{"b":2},{"c":3},{"d":4}]
[true,false,true] [2,1,3,4] [2,1,3,4] [2,1,3,4] [2,1,3,4] [9223372036854775809,9223372036854775808,9223372036854775808,9223372036854775809] [2.0,1.0,3.0,4.0] [2.0,1.0,3.0,4.0] [2016-03-14,2016-03-13,2016-03-12,2016-03-11] [2016-11-15 20:54:00,2016-11-12 20:54:00,2016-11-11 20:54:00] ["a","b","c","d"] [["a","b"],["c","d"],["e"],["f"]] [{"col1":"a","col2":1},{"col1":"b","col2":2},{"col1":"c","col2":3},{"col1":"d","col2":4}] [{"a":1},{"b":2},{"c":3},{"d":4}]


-- !query 13
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ FROM various_maps
-- !query 12 schema
struct<m:map<string,struct<k:string,v1:timestamp,v2:string>>>
-- !query 12 output
{"2016-11-15 20:54:00":{"k":"2016-11-15 20:54:00","v1":2016-11-12 20:54:00.0,"v2":null},"2016-11-15 20:54:00.000":{"k":"2016-11-15 20:54:00.000","v1":null,"v2":"2016-11-12 20:54:00.000"}}
{"2016-11-15 20:54:00":{"k":"2016-11-15 20:54:00","v1":2016-11-12 20:54:00,"v2":null},"2016-11-15 20:54:00.000":{"k":"2016-11-15 20:54:00.000","v1":null,"v2":"2016-11-12 20:54:00.000"}}


-- !query 13
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapconcat.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ FROM various_maps
-- !query 1 schema
struct<boolean_map:map<boolean,boolean>,tinyint_map:map<tinyint,tinyint>,smallint_map:map<smallint,smallint>,int_map:map<int,int>,bigint_map:map<bigint,bigint>,decimal_map:map<decimal(19,0),decimal(19,0)>,float_map:map<float,float>,double_map:map<double,double>,date_map:map<date,date>,timestamp_map:map<timestamp,timestamp>,string_map:map<string,string>,array_map:map<array<string>,array<string>>,struct_map:map<struct<col1:string,col2:int>,struct<col1:string,col2:int>>,string_int_map:map<string,int>,int_string_map:map<int,string>>
-- !query 1 output
{false:true,true:false} {1:2,3:4} {1:2,3:4} {4:6,7:8} {6:7,8:9} {9223372036854775808:9223372036854775809,9223372036854775809:9223372036854775808} {1.0:2.0,3.0:4.0} {1.0:2.0,3.0:4.0} {2016-03-12:2016-03-11,2016-03-14:2016-03-13} {2016-11-11 20:54:00.0:2016-11-09 20:54:00.0,2016-11-15 20:54:00.0:2016-11-12 20:54:00.0} {"a":"b","c":"d"} {["a","b"]:["c","d"],["e"]:["f"]} {{"col1":"a","col2":1}:{"col1":"b","col2":2},{"col1":"c","col2":3}:{"col1":"d","col2":4}} {"a":1,"c":2} {1:"a",2:"c"}
{false:true,true:false} {1:2,3:4} {1:2,3:4} {4:6,7:8} {6:7,8:9} {9223372036854775808:9223372036854775809,9223372036854775809:9223372036854775808} {1.0:2.0,3.0:4.0} {1.0:2.0,3.0:4.0} {2016-03-12:2016-03-11,2016-03-14:2016-03-13} {2016-11-11 20:54:00:2016-11-09 20:54:00,2016-11-15 20:54:00:2016-11-12 20:54:00} {"a":"b","c":"d"} {["a","b"]:["c","d"],["e"]:["f"]} {{"col1":"a","col2":1}:{"col1":"b","col2":2},{"col1":"c","col2":3}:{"col1":"d","col2":4}} {"a":1,"c":2} {1:"a",2:"c"}


-- !query 2
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,4 +150,4 @@ select udf(a), b from values (timestamp('1991-12-06 00:00:00.0'), array(timestam
-- !query 16 schema
struct<CAST(udf(cast(a as string)) AS TIMESTAMP):timestamp,b:array<timestamp>>
-- !query 16 output
1991-12-06 00:00:00 [1991-12-06 01:00:00.0,1991-12-06 12:00:00.0]
1991-12-06 00:00:00 [1991-12-06 01:00:00,1991-12-06 12:00:00]
30 changes: 20 additions & 10 deletions sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,25 +17,35 @@

package org.apache.spark.sql.execution

import java.sql.{Date, Timestamp}

import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession}

class HiveResultSuite extends SharedSparkSession {
import testImplicits._

test("date formatting in hive result") {
val date = "2018-12-28"
val executedPlan = Seq(Date.valueOf(date)).toDS().queryExecution.executedPlan
val result = HiveResult.hiveResultString(executedPlan)
assert(result.head == date)
val dates = Seq("2018-12-28", "1582-10-13", "1582-10-14", "1582-10-15")
val df = dates.toDF("a").selectExpr("cast(a as date) as b")
val executedPlan1 = df.queryExecution.executedPlan
val result = HiveResult.hiveResultString(executedPlan1)
assert(result == dates)
val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
val result2 = HiveResult.hiveResultString(executedPlan2)
assert(result2 == dates.map(x => s"[$x]"))
}

test("timestamp formatting in hive result") {
val timestamp = "2018-12-28 01:02:03"
val executedPlan = Seq(Timestamp.valueOf(timestamp)).toDS().queryExecution.executedPlan
val result = HiveResult.hiveResultString(executedPlan)
assert(result.head == timestamp)
val timestamps = Seq(
"2018-12-28 01:02:03",
"1582-10-13 01:02:03",
"1582-10-14 01:02:03",
"1582-10-15 01:02:03")
val df = timestamps.toDF("a").selectExpr("cast(a as timestamp) as b")
val executedPlan1 = df.queryExecution.executedPlan
val result = HiveResult.hiveResultString(executedPlan1)
assert(result == timestamps)
val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
val result2 = HiveResult.hiveResultString(executedPlan2)
assert(result2 == timestamps.map(x => s"[$x]"))
}

test("toHiveString correctly handles UDTs") {
Expand Down