From 7075138a666114124f1cab064271f2f3e84fb0ef Mon Sep 17 00:00:00 2001 From: Ait Zeouay Amrane Date: Fri, 7 Oct 2022 22:21:35 +0200 Subject: [PATCH 1/2] [SPARK-40705] Handle case of using mutable array when converting Row to JSON --- .../main/scala/org/apache/spark/sql/Row.scala | 2 ++ .../scala/org/apache/spark/sql/RowTest.scala | 25 +++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala index 4f6c9a8c703e..72e1dd94c94d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala @@ -584,6 +584,8 @@ trait Row extends Serializable { case (i: CalendarInterval, _) => JString(i.toString) case (a: Array[_], ArrayType(elementType, _)) => iteratorToJsonArray(a.iterator, elementType) + case (a: mutable.ArraySeq[_], ArrayType(elementType, _)) => + iteratorToJsonArray(a.iterator, elementType) case (s: Seq[_], ArrayType(elementType, _)) => iteratorToJsonArray(s.iterator, elementType) case (m: Map[String @unchecked, _], MapType(StringType, valueType, _)) => diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala index 385f74973684..76848e86c0ee 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala @@ -17,6 +17,9 @@ package org.apache.spark.sql +import scala.collection.mutable.ArraySeq + +import org.json4s.JsonAST.{JArray, JObject, JString} import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.must.Matchers import org.scalatest.matchers.should.Matchers._ @@ -29,8 +32,8 @@ class RowTest extends AnyFunSpec with Matchers { val schema = StructType( StructField("col1", StringType) :: - StructField("col2", StringType) :: - StructField("col3", IntegerType) :: Nil) + StructField("col2", StringType) :: + StructField("col3", IntegerType) :: Nil) val values = Array("value1", "value2", 1) val valuesWithoutCol3 = Array[Any](null, "value2", null) @@ -67,18 +70,12 @@ class RowTest extends AnyFunSpec with Matchers { } it("getValuesMap() retrieves values of multiple fields as a Map(field -> value)") { - val expected = Map( - "col1" -> "value1", - "col2" -> "value2" - ) + val expected = Map("col1" -> "value1", "col2" -> "value2") sampleRow.getValuesMap(List("col1", "col2")) shouldBe expected } it("getValuesMap() retrieves null value on non AnyVal Type") { - val expected = Map( - "col1" -> null, - "col2" -> "value2" - ) + val expected = Map("col1" -> null, "col2" -> "value2") sampleRowWithoutCol3.getValuesMap[String](List("col1", "col2")) shouldBe expected } @@ -91,6 +88,14 @@ class RowTest extends AnyFunSpec with Matchers { it("getAs() on type extending AnyVal does not throw exception when value is null") { sampleRowWithoutCol3.getAs[String](sampleRowWithoutCol3.fieldIndex("col1")) shouldBe null } + + it("json should convert a mutable array to JSON") { + val schema = new StructType().add(StructField("list", ArrayType(StringType))) + val values = ArraySeq("1", "2", "3") + val row = new GenericRowWithSchema(Array(values), schema) + val expectedList = JArray(JString("1") :: JString("2") :: JString("3") :: Nil) + row.jsonValue shouldBe new JObject(("list", expectedList) :: Nil) + } } describe("row equals") { From eb2eb952d28baf6a96a9ddc5e6b878d0750f9435 Mon Sep 17 00:00:00 2001 From: Ait Zeouay Amrane Date: Sat, 8 Oct 2022 17:37:49 +0200 Subject: [PATCH 2/2] [SPARK-40705] revert changes related to Scalafmt --- .../test/scala/org/apache/spark/sql/RowTest.scala | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala index 76848e86c0ee..82731cdb220a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RowTest.scala @@ -32,8 +32,8 @@ class RowTest extends AnyFunSpec with Matchers { val schema = StructType( StructField("col1", StringType) :: - StructField("col2", StringType) :: - StructField("col3", IntegerType) :: Nil) + StructField("col2", StringType) :: + StructField("col3", IntegerType) :: Nil) val values = Array("value1", "value2", 1) val valuesWithoutCol3 = Array[Any](null, "value2", null) @@ -70,12 +70,18 @@ class RowTest extends AnyFunSpec with Matchers { } it("getValuesMap() retrieves values of multiple fields as a Map(field -> value)") { - val expected = Map("col1" -> "value1", "col2" -> "value2") + val expected = Map( + "col1" -> "value1", + "col2" -> "value2" + ) sampleRow.getValuesMap(List("col1", "col2")) shouldBe expected } it("getValuesMap() retrieves null value on non AnyVal Type") { - val expected = Map("col1" -> null, "col2" -> "value2") + val expected = Map( + "col1" -> null, + "col2" -> "value2" + ) sampleRowWithoutCol3.getValuesMap[String](List("col1", "col2")) shouldBe expected }