diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 0532bc163b5d..645d0d709e06 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -218,7 +218,7 @@ statement | (DESC | DESCRIBE) namespace EXTENDED? multipartIdentifier #describeNamespace | (DESC | DESCRIBE) TABLE? option=(EXTENDED | FORMATTED)? - multipartIdentifier partitionSpec? describeColName? #describeTable + multipartIdentifier partitionSpec? describeColName? #describeRelation | (DESC | DESCRIBE) QUERY? query #describeQuery | COMMENT ON namespace multipartIdentifier IS comment=(STRING | NULL) #commentNamespace diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 4ce8fcc3b10c..f7799b781e53 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -759,6 +759,8 @@ class Analyzer( u.failAnalysis(s"${ident.quoted} is a temp view not table.") } u + case u @ UnresolvedTableOrView(ident) => + lookupTempView(ident).map(_ => ResolvedView(ident.asIdentifier)).getOrElse(u) } def lookupTempView(identifier: Seq[String]): Option[LogicalPlan] = { @@ -803,16 +805,16 @@ class Analyzer( .map(ResolvedTable(catalog.asTableCatalog, ident, _)) .getOrElse(u) + case u @ UnresolvedTableOrView(NonSessionCatalogAndIdentifier(catalog, ident)) => + CatalogV2Util.loadTable(catalog, ident) + .map(ResolvedTable(catalog.asTableCatalog, ident, _)) + .getOrElse(u) + case i @ InsertIntoStatement(u: UnresolvedRelation, _, _, _, _) if i.query.resolved => lookupV2Relation(u.multipartIdentifier) .map(v2Relation => i.copy(table = v2Relation)) .getOrElse(i) - case desc @ DescribeTable(u: UnresolvedV2Relation, _) => - CatalogV2Util.loadRelation(u.catalog, u.tableName) - .map(rel => desc.copy(table = rel)) - .getOrElse(desc) - case alter @ AlterTable(_, _, u: UnresolvedV2Relation, _) => CatalogV2Util.loadRelation(u.catalog, u.tableName) .map(rel => alter.copy(table = rel)) @@ -889,17 +891,28 @@ class Analyzer( case u: UnresolvedRelation => lookupRelation(u.multipartIdentifier).map(resolveViews).getOrElse(u) - case u @ UnresolvedTable(identifier: Seq[String]) => - expandRelationName(identifier) match { - case SessionCatalogAndIdentifier(catalog, ident) => - CatalogV2Util.loadTable(catalog, ident) match { - case Some(v1Table: V1Table) if v1Table.v1Table.tableType == CatalogTableType.VIEW => - u.failAnalysis(s"$ident is a view not table.") - case Some(table) => ResolvedTable(catalog.asTableCatalog, ident, table) - case None => u - } - case _ => u - } + case u @ UnresolvedTable(identifier) => + lookupTableOrView(identifier).map { + case v: ResolvedView => + u.failAnalysis(s"${v.identifier.quoted} is a view not table.") + case table => table + }.getOrElse(u) + + case u @ UnresolvedTableOrView(identifier) => + lookupTableOrView(identifier).getOrElse(u) + } + + private def lookupTableOrView(identifier: Seq[String]): Option[LogicalPlan] = { + expandRelationName(identifier) match { + case SessionCatalogAndIdentifier(catalog, ident) => + CatalogV2Util.loadTable(catalog, ident).map { + case v1Table: V1Table if v1Table.v1Table.tableType == CatalogTableType.VIEW => + ResolvedView(ident) + case table => + ResolvedTable(catalog.asTableCatalog, ident, table) + } + case _ => None + } } // Look up a relation from the session catalog with the following logic: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index bd50a6b36efc..1d44c84f3123 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -98,6 +98,9 @@ trait CheckAnalysis extends PredicateHelper { case u: UnresolvedTable => u.failAnalysis(s"Table not found: ${u.multipartIdentifier.quoted}") + case u: UnresolvedTableOrView => + u.failAnalysis(s"Table or view not found: ${u.multipartIdentifier.quoted}") + case u: UnresolvedRelation => u.failAnalysis(s"Table or view not found: ${u.multipartIdentifier.quoted}") @@ -118,13 +121,6 @@ trait CheckAnalysis extends PredicateHelper { case AlterTable(_, _, u: UnresolvedV2Relation, _) => failAnalysis(s"Table not found: ${u.originalNameParts.quoted}") - case DescribeTable(u: UnresolvedV2Relation, _) if isView(u.originalNameParts) => - u.failAnalysis( - s"Invalid command: '${u.originalNameParts.quoted}' is a view not a table.") - - case DescribeTable(u: UnresolvedV2Relation, _) => - failAnalysis(s"Table not found: ${u.originalNameParts.quoted}") - case operator: LogicalPlan => // Check argument data types of higher-order functions downwards first. // If the arguments of the higher-order functions are resolved but the type check fails, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala index 034e30d5a234..b73ba0138906 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala @@ -115,14 +115,6 @@ class ResolveCatalogs(val catalogManager: CatalogManager) } RenameTable(catalog.asTableCatalog, oldName.asIdentifier, newNameParts.asIdentifier) - case DescribeTableStatement( - nameParts @ NonSessionCatalogAndTable(catalog, tbl), partitionSpec, isExtended) => - if (partitionSpec.nonEmpty) { - throw new AnalysisException("DESCRIBE TABLE does not support partition for v2 tables.") - } - val r = UnresolvedV2Relation(nameParts, catalog.asTableCatalog, tbl.asIdentifier) - DescribeTable(r, isExtended) - case DescribeColumnStatement( NonSessionCatalogAndTable(catalog, tbl), colNameParts, isExtended) => throw new AnalysisException("Describing columns is not supported for v2 tables.") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/namespace.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/namespace.scala deleted file mode 100644 index da6955d84991..000000000000 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/namespace.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.catalyst.analysis - -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.logical.LeafNode -import org.apache.spark.sql.connector.catalog.SupportsNamespaces - -case class ResolvedNamespace(catalog: SupportsNamespaces, namespace: Seq[String]) - extends LeafNode { - override def output: Seq[Attribute] = Nil -} - -case class UnresolvedNamespace(multipartIdentifier: Seq[String]) extends LeafNode { - override lazy val resolved: Boolean = false - - override def output: Seq[Attribute] = Nil -} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/table.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/table.scala deleted file mode 100644 index 855e707d9014..000000000000 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/table.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.catalyst.analysis - -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.logical.LeafNode -import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog} - -case class ResolvedTable(catalog: TableCatalog, identifier: Identifier, table: Table) - extends LeafNode { - override def output: Seq[Attribute] = Nil -} - -case class UnresolvedTable(multipartIdentifier: Seq[String]) extends LeafNode { - override lazy val resolved: Boolean = false - - override def output: Seq[Attribute] = Nil -} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala new file mode 100644 index 000000000000..239f987e97a7 --- /dev/null +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.analysis + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} +import org.apache.spark.sql.connector.catalog.{Identifier, SupportsNamespaces, Table, TableCatalog} + +/** + * Holds the name of a namespace that has yet to be looked up in a catalog. It will be resolved to + * [[ResolvedNamespace]] during analysis. + */ +case class UnresolvedNamespace(multipartIdentifier: Seq[String]) extends LeafNode { + override lazy val resolved: Boolean = false + + override def output: Seq[Attribute] = Nil +} + +/** + * Holds the name of a table that has yet to be looked up in a catalog. It will be resolved to + * [[ResolvedTable]] during analysis. + */ +case class UnresolvedTable(multipartIdentifier: Seq[String]) extends LeafNode { + override lazy val resolved: Boolean = false + + override def output: Seq[Attribute] = Nil +} + +/** + * Holds the name of a table or view that has yet to be looked up in a catalog. It will + * be resolved to [[ResolvedTable]] or [[ResolvedView]] during analysis. + */ +case class UnresolvedTableOrView(multipartIdentifier: Seq[String]) extends LeafNode { + override lazy val resolved: Boolean = false + override def output: Seq[Attribute] = Nil +} + +/** + * A plan containing resolved namespace. + */ +case class ResolvedNamespace(catalog: SupportsNamespaces, namespace: Seq[String]) + extends LeafNode { + override def output: Seq[Attribute] = Nil +} + +/** + * A plan containing resolved table. + */ +case class ResolvedTable(catalog: TableCatalog, identifier: Identifier, table: Table) + extends LeafNode { + override def output: Seq[Attribute] = Nil +} + +/** + * A plan containing resolved (temp) views. + */ +// TODO: create a generic representation for temp view, v1 view and v2 view, after we add view +// support to v2 catalog. For now we only need the identifier to fallback to v1 command. +case class ResolvedView(identifier: Identifier) extends LeafNode { + override def output: Seq[Attribute] = Nil +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 7ffb28262647..e326e48da658 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -3062,9 +3062,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } /** - * Create a [[DescribeColumnStatement]] or [[DescribeTableStatement]] commands. + * Create a [[DescribeColumnStatement]] or [[DescribeRelation]] commands. */ - override def visitDescribeTable(ctx: DescribeTableContext): LogicalPlan = withOrigin(ctx) { + override def visitDescribeRelation(ctx: DescribeRelationContext): LogicalPlan = withOrigin(ctx) { val isExtended = ctx.EXTENDED != null || ctx.FORMATTED != null if (ctx.describeColName != null) { if (ctx.partitionSpec != null) { @@ -3086,8 +3086,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } else { Map.empty[String, String] } - DescribeTableStatement( - visitMultipartIdentifier(ctx.multipartIdentifier()), + DescribeRelation( + UnresolvedTableOrView(visitMultipartIdentifier(ctx.multipartIdentifier())), partitionSpec, isExtended) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala index 476ecf5bfdfc..1e097899602a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala @@ -292,14 +292,6 @@ case class DropViewStatement( viewName: Seq[String], ifExists: Boolean) extends ParsedStatement -/** - * A DESCRIBE TABLE tbl_name statement, as parsed from SQL. - */ -case class DescribeTableStatement( - tableName: Seq[String], - partitionSpec: TablePartitionSpec, - isExtended: Boolean) extends ParsedStatement - /** * A DESCRIBE TABLE tbl_name col_name statement, as parsed from SQL. */ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala index f1946dc7bb09..e98b2cf7abfc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.catalyst.analysis.{NamedRelation, UnresolvedException} +import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, Unevaluable} import org.apache.spark.sql.catalyst.plans.DescribeTableSchema import org.apache.spark.sql.connector.catalog._ @@ -314,12 +315,13 @@ case class ShowNamespaces( } /** - * The logical plan of the DESCRIBE TABLE command that works for v2 tables. + * The logical plan of the DESCRIBE relation_name command that works for v2 tables. */ -case class DescribeTable(table: NamedRelation, isExtended: Boolean) extends Command { - - override lazy val resolved: Boolean = table.resolved - +case class DescribeRelation( + relation: LogicalPlan, + partitionSpec: TablePartitionSpec, + isExtended: Boolean) extends Command { + override def children: Seq[LogicalPlan] = Seq(relation) override def output: Seq[Attribute] = DescribeTableSchema.describeTableAttributes() } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala index 86e5894b369a..16aec23521f9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala @@ -98,6 +98,14 @@ private[sql] object CatalogV2Implicits { } def asMultipartIdentifier: Seq[String] = ident.namespace :+ ident.name + + def asTableIdentifier: TableIdentifier = ident.namespace match { + case ns if ns.isEmpty => TableIdentifier(ident.name) + case Array(dbName) => TableIdentifier(ident.name, Some(dbName)) + case _ => + throw new AnalysisException( + s"$quoted is not a valid TableIdentifier as it has more than 2 name parts.") + } } implicit class MultipartIdentifierHelper(parts: Seq[String]) { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala index 53007a23553e..3a4c08235731 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.parser import java.util.Locale import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, GlobalTempView, LocalTempView, PersistedView, UnresolvedAttribute, UnresolvedNamespace, UnresolvedRelation, UnresolvedStar, UnresolvedTable} +import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, GlobalTempView, LocalTempView, PersistedView, UnresolvedAttribute, UnresolvedNamespace, UnresolvedRelation, UnresolvedStar, UnresolvedTable, UnresolvedTableOrView} import org.apache.spark.sql.catalyst.catalog.{ArchiveResource, BucketSpec, FileResource, FunctionResource, FunctionResourceType, JarResource} import org.apache.spark.sql.catalyst.expressions.{EqualTo, Literal} import org.apache.spark.sql.catalyst.plans.logical._ @@ -791,13 +791,13 @@ class DDLParserSuite extends AnalysisTest { test("SPARK-17328 Fix NPE with EXPLAIN DESCRIBE TABLE") { comparePlans(parsePlan("describe t"), - DescribeTableStatement(Seq("t"), Map.empty, isExtended = false)) + DescribeRelation(UnresolvedTableOrView(Seq("t")), Map.empty, isExtended = false)) comparePlans(parsePlan("describe table t"), - DescribeTableStatement(Seq("t"), Map.empty, isExtended = false)) + DescribeRelation(UnresolvedTableOrView(Seq("t")), Map.empty, isExtended = false)) comparePlans(parsePlan("describe table extended t"), - DescribeTableStatement(Seq("t"), Map.empty, isExtended = true)) + DescribeRelation(UnresolvedTableOrView(Seq("t")), Map.empty, isExtended = true)) comparePlans(parsePlan("describe table formatted t"), - DescribeTableStatement(Seq("t"), Map.empty, isExtended = true)) + DescribeRelation(UnresolvedTableOrView(Seq("t")), Map.empty, isExtended = true)) } test("insert table: basic append") { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index c4325289507b..f0c87812daff 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogTableType, CatalogUtils} import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogPlugin, Identifier, LookupCatalog, SupportsNamespaces, Table, TableCatalog, TableChange, V1Table} +import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogPlugin, Identifier, LookupCatalog, SupportsNamespaces, TableCatalog, TableChange, V1Table} import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.execution.command._ import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, RefreshTable} @@ -214,23 +214,12 @@ class ResolveSessionCatalog( case RenameTableStatement(SessionCatalogAndTable(_, oldName), newNameParts, isView) => AlterTableRenameCommand(oldName.asTableIdentifier, newNameParts.asTableIdentifier, isView) - case DescribeTableStatement( - nameParts @ SessionCatalogAndTable(catalog, tbl), partitionSpec, isExtended) => - loadTable(catalog, tbl.asIdentifier).collect { - case v1Table: V1Table => - DescribeTableCommand(tbl.asTableIdentifier, partitionSpec, isExtended) - }.getOrElse { - // The v1 `DescribeTableCommand` can describe view as well. - if (isView(tbl)) { - DescribeTableCommand(tbl.asTableIdentifier, partitionSpec, isExtended) - } else { - if (partitionSpec.nonEmpty) { - throw new AnalysisException("DESCRIBE TABLE does not support partition for v2 tables.") - } - val r = UnresolvedV2Relation(nameParts, catalog.asTableCatalog, tbl.asIdentifier) - DescribeTable(r, isExtended) - } - } + case DescribeRelation(ResolvedTable(_, ident, _: V1Table), partitionSpec, isExtended) => + DescribeTableCommand(ident.asTableIdentifier, partitionSpec, isExtended) + + // Use v1 command to describe (temp) view, as v2 catalog doesn't support view yet. + case DescribeRelation(ResolvedView(ident), partitionSpec, isExtended) => + DescribeTableCommand(ident.asTableIdentifier, partitionSpec, isExtended) case DescribeColumnStatement( SessionCatalogAndTable(catalog, tbl), colNameParts, isExtended) => diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala index 430f76e15b27..b452b66e0381 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala @@ -23,8 +23,8 @@ import org.apache.spark.sql.{AnalysisException, Strategy} import org.apache.spark.sql.catalyst.analysis.{ResolvedNamespace, ResolvedTable} import org.apache.spark.sql.catalyst.expressions.{And, PredicateHelper, SubqueryExpression} import org.apache.spark.sql.catalyst.planning.PhysicalOperation -import org.apache.spark.sql.catalyst.plans.logical.{AlterNamespaceSetLocation, AlterNamespaceSetOwner, AlterNamespaceSetProperties, AlterTable, AppendData, CommentOnNamespace, CommentOnTable, CreateNamespace, CreateTableAsSelect, CreateV2Table, DeleteFromTable, DescribeNamespace, DescribeTable, DropNamespace, DropTable, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, RefreshTable, RenameTable, Repartition, ReplaceTable, ReplaceTableAsSelect, SetCatalogAndNamespace, ShowCurrentNamespace, ShowNamespaces, ShowTableProperties, ShowTables} -import org.apache.spark.sql.connector.catalog.{Identifier, StagingTableCatalog, SupportsNamespaces, TableCapability, TableCatalog, TableChange} +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.connector.catalog.{StagingTableCatalog, SupportsNamespaces, TableCapability, TableCatalog, TableChange} import org.apache.spark.sql.connector.read.streaming.{ContinuousStream, MicroBatchStream} import org.apache.spark.sql.execution.{FilterExec, ProjectExec, SparkPlan} import org.apache.spark.sql.execution.datasources.DataSourceStrategy @@ -197,7 +197,10 @@ object DataSourceV2Strategy extends Strategy with PredicateHelper { case desc @ DescribeNamespace(ResolvedNamespace(catalog, ns), extended) => DescribeNamespaceExec(desc.output, catalog, ns, extended) :: Nil - case desc @ DescribeTable(DataSourceV2Relation(table, _, _), isExtended) => + case desc @ DescribeRelation(ResolvedTable(_, _, table), partitionSpec, isExtended) => + if (partitionSpec.nonEmpty) { + throw new AnalysisException("DESCRIBE does not support partition for v2 tables.") + } DescribeTableExec(desc.output, table, isExtended) :: Nil case DropTable(catalog, ident, ifExists) => diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala index 67d24536a5a4..ddb2926eb6c9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala @@ -39,7 +39,7 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap */ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf) extends TableCatalog with SupportsNamespaces { - import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ + import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper import V2SessionCatalog._ override val defaultNamespace: Array[String] = Array("default") diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 01eff0888e44..1d83717f60b8 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -539,7 +539,8 @@ EXPLAIN EXTENDED DESC t struct -- !query 34 output == Parsed Logical Plan == -'DescribeTableStatement [t], false +'DescribeRelation false ++- 'UnresolvedTableOrView [t] == Analyzed Logical Plan == col_name: string, data_type: string, comment: string diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 9169b3819f0a..dae7d0b38213 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -461,7 +461,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { case _: Join | _: Aggregate | _: Generate | _: Sample | _: Distinct => false case _: DescribeCommandBase | _: DescribeColumnCommand - | _: DescribeTableStatement + | _: DescribeRelation | _: DescribeColumnStatement => true case PhysicalOperation(_, _, Sort(_, true, _)) => true case _ => plan.children.iterator.exists(isSorted) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index e9ec42a8c3b2..5a7774abf092 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -120,6 +120,11 @@ class DataSourceV2SQLSuite Row("", "", ""), Row("# Partitioning", "", ""), Row("Part 0", "id", ""))) + + val e = intercept[AnalysisException] { + sql("DESCRIBE TABLE testcat.table_name PARTITION (id = 1)") + } + assert(e.message.contains("DESCRIBE does not support partition for v2 tables")) } test("DescribeTable with v2 catalog when table does not exist.") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala index 56dceba06d15..06574a9f8fd2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala @@ -18,11 +18,11 @@ package org.apache.spark.sql.execution import org.apache.spark.sql.SaveMode -import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedAlias, UnresolvedAttribute, UnresolvedRelation, UnresolvedStar} import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.expressions.{Ascending, Concat, SortOrder} -import org.apache.spark.sql.catalyst.plans.logical.{DescribeColumnStatement, DescribeTableStatement, LogicalPlan, Project, RepartitionByExpression, Sort} +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, RepartitionByExpression, Sort} import org.apache.spark.sql.execution.command._ import org.apache.spark.sql.execution.datasources.{CreateTable, RefreshResource} import org.apache.spark.sql.internal.{HiveSerDe, SQLConf} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala index 3f95e92e95b6..0901c66cccce 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala @@ -26,11 +26,11 @@ import org.mockito.invocation.InvocationOnMock import org.apache.spark.sql.{AnalysisException, SaveMode} import org.apache.spark.sql.catalyst.{AliasIdentifier, TableIdentifier} -import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, Analyzer, CTESubstitution, EmptyFunctionRegistry, NoSuchTableException, ResolveCatalogs, ResolveSessionCatalog, UnresolvedAttribute, UnresolvedRelation, UnresolvedStar, UnresolvedSubqueryColumnAliases, UnresolvedV2Relation} +import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, Analyzer, CTESubstitution, EmptyFunctionRegistry, NoSuchTableException, ResolveCatalogs, ResolvedTable, ResolveSessionCatalog, UnresolvedAttribute, UnresolvedRelation, UnresolvedStar, UnresolvedSubqueryColumnAliases, UnresolvedV2Relation} import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, InMemoryCatalog, SessionCatalog} import org.apache.spark.sql.catalyst.expressions.{EqualTo, InSubquery, IntegerLiteral, ListQuery, StringLiteral} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser -import org.apache.spark.sql.catalyst.plans.logical.{AlterTable, Assignment, CreateTableAsSelect, CreateV2Table, DeleteAction, DeleteFromTable, DescribeTable, DropTable, InsertAction, LogicalPlan, MergeIntoTable, OneRowRelation, Project, SubqueryAlias, UpdateAction, UpdateTable} +import org.apache.spark.sql.catalyst.plans.logical.{AlterTable, Assignment, CreateTableAsSelect, CreateV2Table, DeleteAction, DeleteFromTable, DescribeRelation, DropTable, InsertAction, LocalRelation, LogicalPlan, MergeIntoTable, OneRowRelation, Project, SubqueryAlias, UpdateAction, UpdateTable} import org.apache.spark.sql.connector.InMemoryTableProvider import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogNotFoundException, Identifier, Table, TableCatalog, TableChange, V1Table} import org.apache.spark.sql.execution.datasources.CreateTable @@ -49,22 +49,20 @@ class PlanResolutionSuite extends AnalysisTest { t } - private val table1: Table = { - val t = mock(classOf[Table]) - when(t.schema()).thenReturn(new StructType().add("i", "int")) - t + private val v1Table: V1Table = { + val t = mock(classOf[CatalogTable]) + when(t.schema).thenReturn(new StructType().add("i", "int")) + when(t.tableType).thenReturn(CatalogTableType.MANAGED) + V1Table(t) } private val testCat: TableCatalog = { val newCatalog = mock(classOf[TableCatalog]) when(newCatalog.loadTable(any())).thenAnswer((invocation: InvocationOnMock) => { invocation.getArgument[Identifier](0).name match { - case "tab" => - table - case "tab1" => - table1 - case name => - throw new NoSuchTableException(name) + case "tab" => table + case "tab1" => table + case name => throw new NoSuchTableException(name) } }) when(newCatalog.name()).thenReturn("testcat") @@ -75,20 +73,11 @@ class PlanResolutionSuite extends AnalysisTest { val newCatalog = mock(classOf[TableCatalog]) when(newCatalog.loadTable(any())).thenAnswer((invocation: InvocationOnMock) => { invocation.getArgument[Identifier](0).name match { - case "v1Table" => - val v1Table = mock(classOf[V1Table]) - when(v1Table.schema).thenReturn(new StructType().add("i", "int")) - v1Table - case "v1Table1" => - val v1Table1 = mock(classOf[V1Table]) - when(v1Table1.schema).thenReturn(new StructType().add("i", "int")) - v1Table1 - case "v2Table" => - table - case "v2Table1" => - table1 - case name => - throw new NoSuchTableException(name) + case "v1Table" => v1Table + case "v1Table1" => v1Table + case "v2Table" => table + case "v2Table1" => table + case name => throw new NoSuchTableException(name) } }) when(newCatalog.name()).thenReturn(CatalogManager.SESSION_CATALOG_NAME) @@ -99,6 +88,7 @@ class PlanResolutionSuite extends AnalysisTest { new InMemoryCatalog, EmptyFunctionRegistry, new SQLConf().copy(SQLConf.CASE_SENSITIVE -> true)) + v1SessionCatalog.createTempView("v", LocalRelation(Nil), false) private val catalogManagerWithDefault = { val manager = mock(classOf[CatalogManager]) @@ -140,10 +130,10 @@ class PlanResolutionSuite extends AnalysisTest { val analyzer = new Analyzer(catalogManager, conf) val rules = Seq( CTESubstitution, + analyzer.ResolveRelations, new ResolveCatalogs(catalogManager), new ResolveSessionCatalog(catalogManager, conf, _ == Seq("v")), - analyzer.ResolveTables, - analyzer.ResolveRelations) + analyzer.ResolveTables) rules.foldLeft(parsePlan(query)) { case (plan, rule) => rule.apply(plan) } @@ -812,7 +802,7 @@ class PlanResolutionSuite extends AnalysisTest { } } - test("DESCRIBE TABLE") { + test("DESCRIBE relation") { Seq("v1Table" -> true, "v2Table" -> false, "testcat.tab" -> false).foreach { case (tblName, useV1Command) => val sql1 = s"DESC TABLE $tblName" @@ -827,27 +817,31 @@ class PlanResolutionSuite extends AnalysisTest { comparePlans(parsed2, expected2) } else { parsed1 match { - case DescribeTable(_: DataSourceV2Relation, isExtended) => + case DescribeRelation(_: ResolvedTable, _, isExtended) => assert(!isExtended) case _ => fail("Expect DescribeTable, but got:\n" + parsed1.treeString) } parsed2 match { - case DescribeTable(_: DataSourceV2Relation, isExtended) => + case DescribeRelation(_: ResolvedTable, _, isExtended) => assert(isExtended) case _ => fail("Expect DescribeTable, but got:\n" + parsed2.treeString) } } val sql3 = s"DESC TABLE $tblName PARTITION(a=1)" + val parsed3 = parseAndResolve(sql3) if (useV1Command) { - val parsed3 = parseAndResolve(sql3) val expected3 = DescribeTableCommand( TableIdentifier(tblName, None), Map("a" -> "1"), false) comparePlans(parsed3, expected3) } else { - val e = intercept[AnalysisException](parseAndResolve(sql3)) - assert(e.message.contains("DESCRIBE TABLE does not support partition for v2 tables")) + parsed3 match { + case DescribeRelation(_: ResolvedTable, partitionSpec, isExtended) => + assert(!isExtended) + assert(partitionSpec == Map("a" -> "1")) + case _ => fail("Expect DescribeTable, but got:\n" + parsed2.treeString) + } } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala index e1615f17a7ba..28e1db961f61 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala @@ -376,7 +376,7 @@ abstract class HiveComparisonTest (!hiveQuery.logical.isInstanceOf[ShowFunctionsStatement]) && (!hiveQuery.logical.isInstanceOf[DescribeFunctionStatement]) && (!hiveQuery.logical.isInstanceOf[DescribeCommandBase]) && - (!hiveQuery.logical.isInstanceOf[DescribeTableStatement]) && + (!hiveQuery.logical.isInstanceOf[DescribeRelation]) && (!hiveQuery.logical.isInstanceOf[DescribeColumnStatement]) && preparedHive != catalyst) {