From 96522ec99064722ce4fbac34b70206feab7f4259 Mon Sep 17 00:00:00 2001 From: Nikola Mandic Date: Tue, 20 Feb 2024 22:20:22 +0100 Subject: [PATCH 1/5] Remove _LEGACY_ERROR_TEMP_1175 from error classes --- .../src/main/resources/error/error-classes.json | 5 ----- .../sql/errors/QueryCompilationErrors.scala | 4 ++-- .../errors/QueryCompilationErrorsSuite.scala | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/common/utils/src/main/resources/error/error-classes.json b/common/utils/src/main/resources/error/error-classes.json index 6c953174865f0..ac3de519bff7f 100644 --- a/common/utils/src/main/resources/error/error-classes.json +++ b/common/utils/src/main/resources/error/error-classes.json @@ -5099,11 +5099,6 @@ "Unrecognized Parquet type: ." ] }, - "_LEGACY_ERROR_TEMP_1175" : { - "message" : [ - "Unsupported data type ." - ] - }, "_LEGACY_ERROR_TEMP_1181" : { "message" : [ "Stream-stream join without equality predicate is not supported." diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index 53338f38ed6d2..08ed584e38b06 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -1908,8 +1908,8 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat def cannotConvertDataTypeToParquetTypeError(field: StructField): Throwable = { new AnalysisException( - errorClass = "_LEGACY_ERROR_TEMP_1175", - messageParameters = Map("dataType" -> field.dataType.catalogString)) + errorClass = "UNSUPPORTED_DATATYPE", + messageParameters = Map("typeName" -> field.dataType.catalogString)) } def incompatibleViewSchemaChangeError( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala index e22399c326f60..86ab854ad7846 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql._ import org.apache.spark.sql.api.java.{UDF1, UDF2, UDF23Test} import org.apache.spark.sql.catalyst.expressions.{Coalesce, Literal, UnsafeRow} import org.apache.spark.sql.catalyst.parser.ParseException +import org.apache.spark.sql.execution.datasources.parquet.SparkToParquetSchemaConverter import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog import org.apache.spark.sql.expressions.SparkUserDefinedFunction import org.apache.spark.sql.functions._ @@ -962,6 +963,22 @@ class QueryCompilationErrorsSuite "methodName" -> "update", "className" -> "org.apache.spark.sql.catalyst.expressions.UnsafeRow")) } + + test("SPARK-42328: Convert unsupported data type from Spark to Parquet") { + val converter = new SparkToParquetSchemaConverter + val dummyDataType = new DataType { + override def defaultSize: Int = 0 + + override private[spark] def asNullable = NullType + } + checkError( + exception = intercept[AnalysisException] { + converter.convertField(StructField("test", dummyDataType)) + }, + errorClass = "UNSUPPORTED_DATATYPE", + parameters = Map("typeName" -> "") + ) + } } class MyCastToString extends SparkUserDefinedFunction( From a7391412949d71099a07796e6f439ec275598a75 Mon Sep 17 00:00:00 2001 From: Nikola Mandic Date: Wed, 21 Feb 2024 17:02:48 +0100 Subject: [PATCH 2/5] Replace _LEGACY_ERROR_TEMP_1175 with INTERNAL_ERROR --- .../sql/errors/QueryCompilationErrors.scala | 5 +++-- .../errors/QueryCompilationErrorsSuite.scala | 17 ----------------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index 08ed584e38b06..38d93f67072cd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -1908,8 +1908,9 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat def cannotConvertDataTypeToParquetTypeError(field: StructField): Throwable = { new AnalysisException( - errorClass = "UNSUPPORTED_DATATYPE", - messageParameters = Map("typeName" -> field.dataType.catalogString)) + errorClass = "INTERNAL_ERROR", + messageParameters = Map("message" -> + "Cannot convert Spark data type ${toSQLType(field.dataType.catalogString)} to Parquet.")) } def incompatibleViewSchemaChangeError( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala index 86ab854ad7846..e22399c326f60 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala @@ -24,7 +24,6 @@ import org.apache.spark.sql._ import org.apache.spark.sql.api.java.{UDF1, UDF2, UDF23Test} import org.apache.spark.sql.catalyst.expressions.{Coalesce, Literal, UnsafeRow} import org.apache.spark.sql.catalyst.parser.ParseException -import org.apache.spark.sql.execution.datasources.parquet.SparkToParquetSchemaConverter import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog import org.apache.spark.sql.expressions.SparkUserDefinedFunction import org.apache.spark.sql.functions._ @@ -963,22 +962,6 @@ class QueryCompilationErrorsSuite "methodName" -> "update", "className" -> "org.apache.spark.sql.catalyst.expressions.UnsafeRow")) } - - test("SPARK-42328: Convert unsupported data type from Spark to Parquet") { - val converter = new SparkToParquetSchemaConverter - val dummyDataType = new DataType { - override def defaultSize: Int = 0 - - override private[spark] def asNullable = NullType - } - checkError( - exception = intercept[AnalysisException] { - converter.convertField(StructField("test", dummyDataType)) - }, - errorClass = "UNSUPPORTED_DATATYPE", - parameters = Map("typeName" -> "") - ) - } } class MyCastToString extends SparkUserDefinedFunction( From d5c10890a5f4bbc532bda3ef1045ad64e62dea10 Mon Sep 17 00:00:00 2001 From: Nikola Mandic Date: Wed, 21 Feb 2024 17:18:24 +0100 Subject: [PATCH 3/5] Update test --- .../sql/errors/QueryCompilationErrors.scala | 2 +- .../errors/QueryCompilationErrorsSuite.scala | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index 38d93f67072cd..829c83f538cf1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -1910,7 +1910,7 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat new AnalysisException( errorClass = "INTERNAL_ERROR", messageParameters = Map("message" -> - "Cannot convert Spark data type ${toSQLType(field.dataType.catalogString)} to Parquet.")) + s"Cannot convert Spark data type ${toSQLType(field.dataType.catalogString)} to Parquet.")) } def incompatibleViewSchemaChangeError( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala index e22399c326f60..51c5552991c40 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql._ import org.apache.spark.sql.api.java.{UDF1, UDF2, UDF23Test} import org.apache.spark.sql.catalyst.expressions.{Coalesce, Literal, UnsafeRow} import org.apache.spark.sql.catalyst.parser.ParseException +import org.apache.spark.sql.execution.datasources.parquet.SparkToParquetSchemaConverter import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog import org.apache.spark.sql.expressions.SparkUserDefinedFunction import org.apache.spark.sql.functions._ @@ -962,6 +963,24 @@ class QueryCompilationErrorsSuite "methodName" -> "update", "className" -> "org.apache.spark.sql.catalyst.expressions.UnsafeRow")) } + + test("INTERNAL_ERROR: Convert unsupported data type from Spark to Parquet") { + val converter = new SparkToParquetSchemaConverter + val dummyDataType = new DataType { + override def defaultSize: Int = 0 + + override def catalogString: String = "Dummy" + + override private[spark] def asNullable = NullType + } + checkError( + exception = intercept[AnalysisException] { + converter.convertField(StructField("test", dummyDataType)) + }, + errorClass = "INTERNAL_ERROR", + parameters = Map("message" -> "Cannot convert Spark data type \"DUMMY\" to Parquet.") + ) + } } class MyCastToString extends SparkUserDefinedFunction( From 4bfa38d9691a7f7190496c76bc54ec711f7b678f Mon Sep 17 00:00:00 2001 From: Nikola Mandic Date: Wed, 21 Feb 2024 21:56:37 +0100 Subject: [PATCH 4/5] Update sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala Co-authored-by: Maxim Gekk --- .../org/apache/spark/sql/errors/QueryCompilationErrors.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index 829c83f538cf1..e96474862b1d7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -1910,7 +1910,7 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat new AnalysisException( errorClass = "INTERNAL_ERROR", messageParameters = Map("message" -> - s"Cannot convert Spark data type ${toSQLType(field.dataType.catalogString)} to Parquet.")) + s"Cannot convert Spark data type ${toSQLType(field.dataType)} to any Parquet type.")) } def incompatibleViewSchemaChangeError( From 7905a5e98f0e04d3cb6c6fd600832ac8e56bf555 Mon Sep 17 00:00:00 2001 From: Nikola Mandic Date: Wed, 21 Feb 2024 21:59:30 +0100 Subject: [PATCH 5/5] Update test --- .../apache/spark/sql/errors/QueryCompilationErrorsSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala index 51c5552991c40..d4e4a41155eaf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala @@ -969,7 +969,7 @@ class QueryCompilationErrorsSuite val dummyDataType = new DataType { override def defaultSize: Int = 0 - override def catalogString: String = "Dummy" + override def simpleString: String = "Dummy" override private[spark] def asNullable = NullType } @@ -978,7 +978,7 @@ class QueryCompilationErrorsSuite converter.convertField(StructField("test", dummyDataType)) }, errorClass = "INTERNAL_ERROR", - parameters = Map("message" -> "Cannot convert Spark data type \"DUMMY\" to Parquet.") + parameters = Map("message" -> "Cannot convert Spark data type \"DUMMY\" to any Parquet type.") ) } }