From 3aa737160c27bfb8845a9d6c880451db88baaff2 Mon Sep 17 00:00:00 2001 From: Andrew Ray Date: Mon, 5 Dec 2016 15:41:28 -0600 Subject: [PATCH 1/4] the fix --- .../scala/org/apache/spark/sql/catalyst/ScalaReflection.scala | 2 +- .../org/apache/spark/sql/catalyst/encoders/RowEncoder.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala index 6e20096901d9..ad218cf88de2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala @@ -342,7 +342,7 @@ object ScalaReflection extends ScalaReflection { StaticInvoke( ArrayBasedMapData.getClass, - ObjectType(classOf[Map[_, _]]), + ObjectType(classOf[scala.collection.immutable.Map[_, _]]), "toScalaMap", keyData :: valueData :: Nil) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala index e95e97b9dc6c..3c689845c2f3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala @@ -288,7 +288,7 @@ object RowEncoder { StaticInvoke( ArrayBasedMapData.getClass, - ObjectType(classOf[Map[_, _]]), + ObjectType(classOf[scala.collection.immutable.Map[_, _]]), "toScalaMap", keyData :: valueData :: Nil) From c0705094151ebefc7a43299bc6d07116a425aa35 Mon Sep 17 00:00:00 2001 From: Andrew Ray Date: Mon, 5 Dec 2016 17:02:48 -0600 Subject: [PATCH 2/4] unit test --- .../scala/org/apache/spark/sql/DatasetSuite.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index d31c766cb779..c0aa795f2550 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -1110,8 +1110,20 @@ class DatasetSuite extends QueryTest with SharedSQLContext { } assert(e.getMessage.contains("Cannot create encoder for Option of Product type")) } + + test("SPARK-18717: code generation works for both scala.collection.Map" + + " and scala.collection.imutable.Map") { + val ds = Seq(WithImmutableMap("hi", Map(42L -> "foo"))).toDS + checkDataset(ds.map(t => t), WithImmutableMap("hi", Map(42L -> "foo"))) + + val ds2 = Seq(WithMap("hi", Map(42L -> "foo"))).toDS + checkDataset(ds2.map(t => t), WithMap("hi", Map(42L -> "foo"))) + } } +case class WithImmutableMap(id: String, map_test: scala.collection.immutable.Map[Long, String]) +case class WithMap(id: String, map_test: scala.collection.Map[Long, String]) + case class Generic[T](id: T, value: Double) case class OtherTuple(_1: String, _2: Int) From 699cc75f8e5d0d53225bd7a8003eed949a8250e0 Mon Sep 17 00:00:00 2001 From: Andrew Ray Date: Mon, 5 Dec 2016 17:07:23 -0600 Subject: [PATCH 3/4] unused import --- .../org/apache/spark/sql/catalyst/encoders/RowEncoder.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala index 3c689845c2f3..bd17f5c91e4a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.catalyst.encoders -import scala.collection.Map import scala.reflect.ClassTag import org.apache.spark.SparkException From f325e75ab61ac903a605090b756461f4b50b64ff Mon Sep 17 00:00:00 2001 From: Andrew Ray Date: Tue, 6 Dec 2016 12:42:18 -0600 Subject: [PATCH 4/4] revert changes to RowEncoder.scala --- .../org/apache/spark/sql/catalyst/encoders/RowEncoder.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala index bd17f5c91e4a..e95e97b9dc6c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.catalyst.encoders +import scala.collection.Map import scala.reflect.ClassTag import org.apache.spark.SparkException @@ -287,7 +288,7 @@ object RowEncoder { StaticInvoke( ArrayBasedMapData.getClass, - ObjectType(classOf[scala.collection.immutable.Map[_, _]]), + ObjectType(classOf[Map[_, _]]), "toScalaMap", keyData :: valueData :: Nil)