From b0fbd5ee98de0ed725d96e63943da934150f6c3e Mon Sep 17 00:00:00 2001 From: aokolnychyi Date: Wed, 26 Jul 2017 21:56:44 +0200 Subject: [PATCH 1/3] [SPARK-21538][SQL] Fix attribute resolution inconsistency in the Dataset API --- sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala index 71ab0ddf2d6f..5db732b19b9e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala @@ -1107,7 +1107,7 @@ class Dataset[T] private[sql]( */ @scala.annotation.varargs def sort(sortCol: String, sortCols: String*): Dataset[T] = { - sort((sortCol +: sortCols).map(apply) : _*) + sort((sortCol +: sortCols).map(Column(_)) : _*) } /** From 309cb8f09f5af81f230911574274c0ca9eb65f34 Mon Sep 17 00:00:00 2001 From: aokolnychyi Date: Thu, 27 Jul 2017 18:54:50 +0200 Subject: [PATCH 2/3] [SPARK-21538][SQL] Added a test case --- .../test/scala/org/apache/spark/sql/DatasetSuite.scala | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index 73098cdb9247..042d66c40509 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -1304,6 +1304,15 @@ class DatasetSuite extends QueryTest with SharedSQLContext { assert(rlike3.count() == 0) } } + + test("SPARK-21538: Attribute resolution inconsistency in Dataset API") { + val df = spark.range(1).withColumnRenamed("id", "x") + checkAnswer(df.sort(col("id")), df.sort("id")) + checkAnswer(df.sort($"id"), df.sort("id")) + checkAnswer(df.sort('id), df.sort("id")) + checkAnswer(df.orderBy('id), df.sort("id")) + checkAnswer(df.orderBy("id"), df.sort("id")) + } } case class WithImmutableMap(id: String, map_test: scala.collection.immutable.Map[Long, String]) From 0b0eea941cb850967e943719822cfab89479a025 Mon Sep 17 00:00:00 2001 From: aokolnychyi Date: Thu, 27 Jul 2017 20:29:24 +0200 Subject: [PATCH 3/3] [SPARK-21538][SQL] Updated test case --- .../org/apache/spark/sql/DatasetSuite.scala | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index 042d66c40509..40235e32d35d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -1306,12 +1306,16 @@ class DatasetSuite extends QueryTest with SharedSQLContext { } test("SPARK-21538: Attribute resolution inconsistency in Dataset API") { - val df = spark.range(1).withColumnRenamed("id", "x") - checkAnswer(df.sort(col("id")), df.sort("id")) - checkAnswer(df.sort($"id"), df.sort("id")) - checkAnswer(df.sort('id), df.sort("id")) - checkAnswer(df.orderBy('id), df.sort("id")) - checkAnswer(df.orderBy("id"), df.sort("id")) + val df = spark.range(3).withColumnRenamed("id", "x") + val expected = Row(0) :: Row(1) :: Row (2) :: Nil + checkAnswer(df.sort("id"), expected) + checkAnswer(df.sort(col("id")), expected) + checkAnswer(df.sort($"id"), expected) + checkAnswer(df.sort('id), expected) + checkAnswer(df.orderBy("id"), expected) + checkAnswer(df.orderBy(col("id")), expected) + checkAnswer(df.orderBy($"id"), expected) + checkAnswer(df.orderBy('id), expected) } }