From 626f26598ccfc692cca59e29a2d7861133654ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=8D=E5=86=AC?= Date: Wed, 29 Aug 2018 11:14:11 +0800 Subject: [PATCH 1/4] Fix exhaustive match using collect --- .../org/apache/spark/sql/execution/ProjectionOverSchema.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala index 2236f18b0da12..7966e772cf1c6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala @@ -38,7 +38,7 @@ private[execution] case class ProjectionOverSchema(schema: StructType) { case GetArrayItem(child, arrayItemOrdinal) => getProjection(child).map { projection => GetArrayItem(projection, arrayItemOrdinal) } case a: GetArrayStructFields => - getProjection(a.child).map(p => (p, p.dataType)).map { + getProjection(a.child).map(p => (p, p.dataType)).collect { case (projection, ArrayType(projSchema @ StructType(_), _)) => GetArrayStructFields(projection, projSchema(a.field.name), @@ -49,7 +49,7 @@ private[execution] case class ProjectionOverSchema(schema: StructType) { case GetMapValue(child, key) => getProjection(child).map { projection => GetMapValue(projection, key) } case GetStructFieldObject(child, field: StructField) => - getProjection(child).map(p => (p, p.dataType)).map { + getProjection(child).map(p => (p, p.dataType)).collect { case (projection, projSchema: StructType) => GetStructField(projection, projSchema.fieldIndex(field.name)) } From ffdb12d4ffafd9ae0f48469e2f4db9e27444e724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=8D=E5=86=AC?= Date: Wed, 29 Aug 2018 15:19:39 +0800 Subject: [PATCH 2/4] scala.util.Properties is more general --- .../scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index aa5b531992613..a676cf6ce6925 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.hive import java.io.{BufferedWriter, File, FileWriter} -import scala.tools.nsc.Properties +import scala.util.Properties import org.apache.hadoop.fs.Path import org.scalatest.{BeforeAndAfterEach, Matchers} From f30f47d0fa7593ef15f348c3f4770c1f39481f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=8D=E5=86=AC?= Date: Thu, 30 Aug 2018 10:19:43 +0800 Subject: [PATCH 3/4] same semantics with an IllegalStateException --- .../spark/sql/execution/ProjectionOverSchema.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala index 7966e772cf1c6..9daf72efd0dbd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala @@ -45,13 +45,21 @@ private[execution] case class ProjectionOverSchema(schema: StructType) { projSchema.fieldIndex(a.field.name), projSchema.size, a.containsNull) + case (_, projSchema) => + throw new IllegalStateException( + s"unmatched child schema for GetArrayStructFields: ${projSchema.toString}" + ) } case GetMapValue(child, key) => getProjection(child).map { projection => GetMapValue(projection, key) } case GetStructFieldObject(child, field: StructField) => - getProjection(child).map(p => (p, p.dataType)).collect { + getProjection(child).map(p => (p, p.dataType)).map { case (projection, projSchema: StructType) => GetStructField(projection, projSchema.fieldIndex(field.name)) + case (_, projSchema) => + throw new IllegalStateException( + s"unmatched child schema for GetStructField: ${projSchema.toString}" + ) } case _ => None From 2dff17b8288a8c3cd82b2d3514bfc2d1f06d88d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=8D=E5=86=AC?= Date: Thu, 30 Aug 2018 10:21:39 +0800 Subject: [PATCH 4/4] fix --- .../org/apache/spark/sql/execution/ProjectionOverSchema.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala index 9daf72efd0dbd..612a7b87b9832 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ProjectionOverSchema.scala @@ -38,7 +38,7 @@ private[execution] case class ProjectionOverSchema(schema: StructType) { case GetArrayItem(child, arrayItemOrdinal) => getProjection(child).map { projection => GetArrayItem(projection, arrayItemOrdinal) } case a: GetArrayStructFields => - getProjection(a.child).map(p => (p, p.dataType)).collect { + getProjection(a.child).map(p => (p, p.dataType)).map { case (projection, ArrayType(projSchema @ StructType(_), _)) => GetArrayStructFields(projection, projSchema(a.field.name),