From ff9c2c2a0a5032cc7f6eb8a8bda3177330fa8938 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 4 Jan 2024 14:50:19 +0800 Subject: [PATCH 1/4] init --- .../spark/sql/catalyst/expressions/literals.scala | 3 ++- .../catalyst/expressions/LiteralExpressionSuite.scala | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index c406ba0707b3a..79b2985adc1de 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -32,7 +32,7 @@ import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period, ZoneOffse import java.util import java.util.Objects -import scala.collection.mutable +import scala.collection.{immutable, mutable} import scala.math.{BigDecimal, BigInt} import scala.reflect.runtime.universe.TypeTag import scala.util.Try @@ -91,6 +91,7 @@ object Literal { case p: Period => Literal(periodToMonths(p), YearMonthIntervalType()) case a: Array[Byte] => Literal(a, BinaryType) case a: mutable.ArraySeq[_] => apply(a.array) + case a: immutable.ArraySeq[_] => apply(a.unsafeArray) case a: Array[_] => val elementType = componentTypeToDataType(a.getClass.getComponentType()) val dataType = ArrayType(elementType) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala index 0a1acd06bd9e7..8e9c9ee86b5c8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala @@ -22,7 +22,7 @@ import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period, ZoneOffse import java.time.temporal.ChronoUnit import java.util.TimeZone -import scala.collection.mutable +import scala.collection.{immutable, mutable} import scala.reflect.runtime.universe.TypeTag import org.apache.spark.{SparkException, SparkFunSuite} @@ -477,4 +477,12 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { Literal.create(UTF8String.fromString("Spark SQL"), ObjectType(classOf[UTF8String])), UTF8String.fromString("Spark SQL")) } + + test("SPARK-46604: Literal support immutable ArraySeq") { + val immArraySeq = immutable.ArraySeq.unsafeWrapArray(Array(1.0, 4.0)) + val expected = toCatalyst(immArraySeq) + checkEvaluation(Literal(immArraySeq), expected) + checkEvaluation(Literal.create(immArraySeq), expected) + checkEvaluation(Literal.create(immArraySeq, ArrayType(DoubleType)), expected) + } } From 5a959f26bec4d55aacc3274efce83d84e8195ee7 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sat, 6 Jan 2024 00:09:54 +0800 Subject: [PATCH 2/4] use toImmutableArraySeq --- .../sql/catalyst/expressions/LiteralExpressionSuite.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala index 8e9c9ee86b5c8..df04b9a1f477d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala @@ -479,7 +479,8 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { } test("SPARK-46604: Literal support immutable ArraySeq") { - val immArraySeq = immutable.ArraySeq.unsafeWrapArray(Array(1.0, 4.0)) + import org.apache.spark.util.ArrayImplicits._ + val immArraySeq = Array(1.0, 4.0).toImmutableArraySeq val expected = toCatalyst(immArraySeq) checkEvaluation(Literal(immArraySeq), expected) checkEvaluation(Literal.create(immArraySeq), expected) From 1169ceb22d6a06fcbda3a8bb513e2b5924903f42 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sat, 6 Jan 2024 00:21:49 +0800 Subject: [PATCH 3/4] revert --- .../sql/catalyst/expressions/LiteralExpressionSuite.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala index df04b9a1f477d..8e9c9ee86b5c8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala @@ -479,8 +479,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { } test("SPARK-46604: Literal support immutable ArraySeq") { - import org.apache.spark.util.ArrayImplicits._ - val immArraySeq = Array(1.0, 4.0).toImmutableArraySeq + val immArraySeq = immutable.ArraySeq.unsafeWrapArray(Array(1.0, 4.0)) val expected = toCatalyst(immArraySeq) checkEvaluation(Literal(immArraySeq), expected) checkEvaluation(Literal.create(immArraySeq), expected) From 813f6975842bf8816c33eb4d7069183cdc38514b Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sat, 6 Jan 2024 00:24:10 +0800 Subject: [PATCH 4/4] use toImmutableArraySeq --- .../sql/catalyst/expressions/LiteralExpressionSuite.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala index 8e9c9ee86b5c8..f63b60f5ebba1 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala @@ -22,7 +22,7 @@ import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period, ZoneOffse import java.time.temporal.ChronoUnit import java.util.TimeZone -import scala.collection.{immutable, mutable} +import scala.collection.mutable import scala.reflect.runtime.universe.TypeTag import org.apache.spark.{SparkException, SparkFunSuite} @@ -479,7 +479,8 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { } test("SPARK-46604: Literal support immutable ArraySeq") { - val immArraySeq = immutable.ArraySeq.unsafeWrapArray(Array(1.0, 4.0)) + import org.apache.spark.util.ArrayImplicits._ + val immArraySeq = Array(1.0, 4.0).toImmutableArraySeq val expected = toCatalyst(immArraySeq) checkEvaluation(Literal(immArraySeq), expected) checkEvaluation(Literal.create(immArraySeq), expected)