diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 097307bce3de..962b6b88f56d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -3479,7 +3479,7 @@ class Analyzer(override val catalogManager: CatalogManager) i.userSpecifiedCols.map { col => i.table.resolve(Seq(col), resolver).getOrElse { - val candidates = i.table.output.map(_.name) + val candidates = i.table.output.map(_.qualifiedName) val orderedCandidates = StringUtils.orderStringsBySimilarity(col, candidates) throw QueryCompilationErrors .unresolvedAttributeError("UNRESOLVED_COLUMN", col, orderedCandidates, i.origin) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala index 4181edcb8c60..99e5f411bdb6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala @@ -71,11 +71,11 @@ trait NamedExpression extends Expression { def exprId: ExprId /** - * Returns a dot separated fully qualified name for this attribute. Given that there can be - * multiple qualifiers, it is possible that there are other possible way to refer to this - * attribute. + * Returns a dot separated fully qualified name for this attribute. If the name or any qualifier + * contains `dots`, it is quoted to avoid confusion. Given that there can be multiple qualifiers, + * it is possible that there are other possible way to refer to this attribute. */ - def qualifiedName: String = (qualifier :+ name).mkString(".") + def qualifiedName: String = (qualifier :+ name).map(quoteIfNeeded).mkString(".") /** * Optional qualifier for the expression. diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala index ac823183ce9d..88de52aec8f6 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala @@ -876,6 +876,26 @@ class AnalysisErrorSuite extends AnalysisTest { "[`a`, `b`, `c`, `d`, `e`]" :: Nil) + errorClassTest( + "SPARK-39783: backticks in error message for candidate column with dots", + // This selects a column that does not exist, + // the error message suggest the existing column with correct backticks + testRelation6.select($"`the`.`id`"), + errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", + messageParameters = Map( + "objectName" -> "`the`.`id`", + "proposal" -> "`the.id`")) + + errorClassTest( + "SPARK-39783: backticks in error message for candidate struct column", + // This selects a column that does not exist, + // the error message suggest the existing column with correct backticks + nestedRelation2.select($"`top.aField`"), + errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", + messageParameters = Map( + "objectName" -> "`top.aField`", + "proposal" -> "`top`")) + test("SPARK-35080: Unsupported correlated equality predicates in subquery") { val a = AttributeReference("a", IntegerType)() val b = AttributeReference("b", IntegerType)() diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala index 33b602907093..d54237fcc140 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala @@ -46,6 +46,8 @@ object TestRelations { val testRelation5 = LocalRelation(AttributeReference("i", StringType)()) + val testRelation6 = LocalRelation(AttributeReference("the.id", LongType)()) + val nestedRelation = LocalRelation( AttributeReference("top", StructType( StructField("duplicateField", StringType) :: diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index 591d2b943590..260cc9396ab9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -1992,6 +1992,51 @@ class DatasetSuite extends QueryTest } } + test("SPARK-39783: Fix error messages for columns with dots/periods") { + forAll(dotColumnTestModes) { (caseSensitive, colName) => + val ds = Seq(SpecialCharClass("1", "2")).toDS + withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive) { + checkError( + exception = intercept[AnalysisException] { + // Note: ds(colName) "SPARK-25153: Improve error messages for columns with dots/periods" + // has different semantics than ds.select(colName) + ds.select(colName) + }, + errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", + sqlState = None, + parameters = Map( + "objectName" -> s"`${colName.replace(".", "`.`")}`", + "proposal" -> "`field.1`, `field 2`")) + } + } + } + + test("SPARK-39783: backticks in error message for candidate column with dots") { + checkError( + exception = intercept[AnalysisException] { + Seq(0).toDF("the.id").select("the.id") + }, + errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", + sqlState = None, + parameters = Map( + "objectName" -> "`the`.`id`", + "proposal" -> "`the.id`")) + } + + test("SPARK-39783: backticks in error message for map candidate key with dots") { + checkError( + exception = intercept[AnalysisException] { + spark.range(1) + .select(map(lit("key"), lit(1)).as("map"), lit(2).as("other.column")) + .select($"`map`"($"nonexisting")).show() + }, + errorClass = "UNRESOLVED_MAP_KEY.WITH_SUGGESTION", + sqlState = None, + parameters = Map( + "objectName" -> "`nonexisting`", + "proposal" -> "`map`, `other.column`")) + } + test("groupBy.as") { val df1 = Seq(DoubleData(1, "one"), DoubleData(2, "two"), DoubleData(3, "three")).toDS() .repartition($"id").sortWithinPartitions("id") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetUnpivotSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetUnpivotSuite.scala index f2f31851acba..3c05a7415a12 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetUnpivotSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetUnpivotSuite.scala @@ -504,10 +504,9 @@ class DatasetUnpivotSuite extends QueryTest checkError( exception = e, errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", - // expected message is wrong: https://issues.apache.org/jira/browse/SPARK-39783 parameters = Map( "objectName" -> "`an`.`id`", - "proposal" -> "`an`.`id`, `int1`, `long1`, `str`.`one`, `str`.`two`")) + "proposal" -> "`an.id`, `int1`, `long1`, `str.one`, `str.two`")) } test("unpivot with struct fields") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala index 75c157d4b885..8086a0e97ec2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala @@ -419,6 +419,22 @@ class QueryCompilationErrorsSuite ) } + test("UNRESOLVED_MAP_KEY: proposal columns containing quoted dots") { + val query = "select m[a] from (select map('a', 'b') as m, 'aa' as `a.a`)" + checkError( + exception = intercept[AnalysisException] {sql(query)}, + errorClass = "UNRESOLVED_MAP_KEY.WITH_SUGGESTION", + sqlState = None, + parameters = Map("objectName" -> "`a`", + "proposal" -> + "`__auto_generated_subquery_name`.`m`, `__auto_generated_subquery_name`.`a.a`"), + context = ExpectedContext( + fragment = "a", + start = 9, + stop = 9) + ) + } + test("UNRESOLVED_COLUMN: SELECT distinct does not work correctly " + "if order by missing attribute") { checkAnswer(