diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index 36dccf92ebf4..440d380ac5cd 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -72,6 +72,8 @@ license: | - In Spark 3.0.2, `PARTITION(col=null)` is always parsed as a null literal in the partition spec. In Spark 3.0.1 or earlier, it is parsed as a string literal of its text representation, e.g., string "null", if the partition column is string type. To restore the legacy behavior, you can set `spark.sql.legacy.parseNullPartitionSpecAsStringLiteral` as true. + - In Spark 3.0.0, the output schema of `SHOW DATABASES` becomes `namespace: string`. In Spark version 2.4 and earlier, the schema was `databaseName: string`. Since Spark 3.0.2, you can restore the old schema by setting `spark.sql.legacy.keepCommandOutputSchema` to `true`. + ## Upgrading from Spark SQL 3.0 to 3.0.1 - In Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Since version 3.0.1, the timestamp type inference is disabled by default. Set the JSON option `inferTimestamp` to `true` to enable such type inference. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala index 67056470418f..6f7fd88f5214 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala @@ -321,11 +321,14 @@ case class AlterNamespaceSetLocation( */ case class ShowNamespaces( namespace: LogicalPlan, - pattern: Option[String]) extends Command { + pattern: Option[String], + override val output: Seq[Attribute] = ShowNamespaces.OUTPUT) extends Command { override def children: Seq[LogicalPlan] = Seq(namespace) + override def producedAttributes: AttributeSet = outputSet +} - override val output: Seq[Attribute] = Seq( - AttributeReference("namespace", StringType, nullable = false)()) +object ShowNamespaces { + val OUTPUT = Seq(AttributeReference("namespace", StringType, nullable = false)()) } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 687f645714f1..55c70165be40 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -3017,6 +3017,15 @@ object SQLConf { .booleanConf .createWithDefault(false) + val LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA = + buildConf("spark.sql.legacy.keepCommandOutputSchema") + .internal() + .doc("When true, Spark will keep the output schema of commands such as SHOW DATABASES " + + "unchanged, for v1 catalog and/or table.") + .version("3.0.2") + .booleanConf + .createWithDefault(false) + /** * Holds information about keys that have been deprecated. * diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index 62a97f49ce9c..53da69e709e9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -237,6 +237,14 @@ class ResolveSessionCatalog( } AlterDatabaseSetLocationCommand(ns.head, location) + case s @ ShowNamespaces(ResolvedNamespace(cata, _), _, output) if isSessionCatalog(cata) => + if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) { + assert(output.length == 1) + s.copy(output = Seq(output.head.withName("databaseName"))) + } else { + s + } + // v1 RENAME TABLE supports temp view. case RenameTableStatement(TempViewOrV1Table(oldName), newName, isView) => AlterTableRenameCommand(oldName.asTableIdentifier, newName.asTableIdentifier, isView) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala index dffe0a838d80..643607a905d7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala @@ -315,8 +315,8 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat case DropNamespace(ResolvedNamespace(catalog, ns), ifExists, cascade) => DropNamespaceExec(catalog, ns, ifExists, cascade) :: Nil - case r @ ShowNamespaces(ResolvedNamespace(catalog, ns), pattern) => - ShowNamespacesExec(r.output, catalog.asNamespaceCatalog, ns, pattern) :: Nil + case ShowNamespaces(ResolvedNamespace(catalog, ns), pattern, output) => + ShowNamespacesExec(output, catalog.asNamespaceCatalog, ns, pattern) :: Nil case r @ ShowTables(ResolvedNamespace(catalog, ns), pattern) => ShowTablesExec(r.output, catalog.asTableCatalog, ns, pattern) :: Nil diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 64c25663bb57..5c08b066019b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1324,6 +1324,12 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { Nil) } + test("SPARK-34359: keep the legacy output schema") { + withSQLConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA.key -> "true") { + assert(sql("SHOW NAMESPACES").schema.fieldNames.toSeq == Seq("databaseName")) + } + } + test("drop view - temporary view") { val catalog = spark.sessionState.catalog sql(