diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index 8049447cd3b5..93800318c232 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -105,6 +105,8 @@ license: | - In Spark 3.0.2, `PARTITION(col=null)` is always parsed as a null literal in the partition spec. In Spark 3.0.1 or earlier, it is parsed as a string literal of its text representation, e.g., string "null", if the partition column is string type. To restore the legacy behavior, you can set `spark.sql.legacy.parseNullPartitionSpecAsStringLiteral` as true. + - In Spark 3.0.0, the output schema of `SHOW DATABASES` becomes `namespace: string`. In Spark version 2.4 and earlier, the schema was `databaseName: string`. Since Spark 3.0.2, you can restore the old schema by setting `spark.sql.legacy.keepCommandOutputSchema` to `true`. + ## Upgrading from Spark SQL 3.0 to 3.0.1 - In Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Since version 3.0.1, the timestamp type inference is disabled by default. Set the JSON option `inferTimestamp` to `true` to enable such type inference. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala index 9c02ffefeab3..6dab23c99f79 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala @@ -325,11 +325,13 @@ case class AlterNamespaceSetLocation( */ case class ShowNamespaces( namespace: LogicalPlan, - pattern: Option[String]) extends Command { + pattern: Option[String], + override val output: Seq[Attribute] = ShowNamespaces.OUTPUT) extends Command { override def children: Seq[LogicalPlan] = Seq(namespace) +} - override val output: Seq[Attribute] = Seq( - AttributeReference("namespace", StringType, nullable = false)()) +object ShowNamespaces { + val OUTPUT = Seq(AttributeReference("namespace", StringType, nullable = false)()) } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index ad242d8d0e4d..278d5482d1e9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -3069,6 +3069,15 @@ object SQLConf { .booleanConf .createWithDefault(false) + val LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA = + buildConf("spark.sql.legacy.keepCommandOutputSchema") + .internal() + .doc("When true, Spark will keep the output schema of commands such as SHOW DATABASES " + + "unchanged, for v1 catalog and/or table.") + .version("3.0.2") + .booleanConf + .createWithDefault(false) + /** * Holds information about keys that have been deprecated. * diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index d7964bdb0bef..922c5fe18827 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -188,6 +188,14 @@ class ResolveSessionCatalog(val catalogManager: CatalogManager) case AlterNamespaceSetLocation(DatabaseInSessionCatalog(db), location) => AlterDatabaseSetLocationCommand(db, location) + case s @ ShowNamespaces(ResolvedNamespace(cata, _), _, output) if isSessionCatalog(cata) => + if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) { + assert(output.length == 1) + s.copy(output = Seq(output.head.withName("databaseName"))) + } else { + s + } + case RenameTable(ResolvedV1TableOrViewIdentifier(oldName), newName, isView) => AlterTableRenameCommand(oldName.asTableIdentifier, newName.asTableIdentifier, isView) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala index f38b2a585055..408c3112fcaf 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala @@ -334,8 +334,8 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat case DropNamespace(ResolvedNamespace(catalog, ns), ifExists, cascade) => DropNamespaceExec(catalog, ns, ifExists, cascade) :: Nil - case r @ ShowNamespaces(ResolvedNamespace(catalog, ns), pattern) => - ShowNamespacesExec(r.output, catalog.asNamespaceCatalog, ns, pattern) :: Nil + case ShowNamespaces(ResolvedNamespace(catalog, ns), pattern, output) => + ShowNamespacesExec(output, catalog.asNamespaceCatalog, ns, pattern) :: Nil case r @ ShowTables(ResolvedNamespace(catalog, ns), pattern) => ShowTablesExec(r.output, catalog.asTableCatalog, ns, pattern) :: Nil diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala index fd76ef2490f3..52742a2f2332 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala @@ -38,6 +38,12 @@ trait ShowNamespacesSuiteBase extends command.ShowNamespacesSuiteBase { }.getMessage assert(errMsg.contains("Namespace 'dummy' not found")) } + + test("SPARK-34359: keep the legacy output schema") { + withSQLConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA.key -> "true") { + assert(sql("SHOW NAMESPACES").schema.fieldNames.toSeq == Seq("databaseName")) + } + } } class ShowNamespacesSuite extends ShowNamespacesSuiteBase with CommandSuiteBase {