Skip to content

Commit d44f56e

Browse files
imback82imback82
committed
initial commit
1 parent f9cc7fb commit d44f56e

File tree

8 files changed

+262
-82
lines changed

8 files changed

+262
-82
lines changed

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1193,15 +1193,6 @@ class DDLParserSuite extends AnalysisTest {
11931193
"DESC TABLE COLUMN for a specific partition is not supported"))
11941194
}
11951195

1196-
test("describe database") {
1197-
val sql1 = "DESCRIBE DATABASE EXTENDED a.b"
1198-
val sql2 = "DESCRIBE DATABASE a.b"
1199-
comparePlans(parsePlan(sql1),
1200-
DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = true))
1201-
comparePlans(parsePlan(sql2),
1202-
DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = false))
1203-
}
1204-
12051196
test("SPARK-17328 Fix NPE with EXPLAIN DESCRIBE TABLE") {
12061197
comparePlans(parsePlan("describe t"),
12071198
DescribeRelation(

‎sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala‎

Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1233,26 +1233,6 @@ class DataSourceV2SQLSuite
12331233
assert(exception.getMessage.contains("Namespace 'ns1' not found"))
12341234
}
12351235

1236-
test("DescribeNamespace using v2 catalog") {
1237-
withNamespace("testcat.ns1.ns2") {
1238-
sql("CREATE NAMESPACE IF NOT EXISTS testcat.ns1.ns2 COMMENT " +
1239-
"'test namespace' LOCATION '/tmp/ns_test'")
1240-
val descriptionDf = sql("DESCRIBE NAMESPACE testcat.ns1.ns2")
1241-
assert(descriptionDf.schema.map(field => (field.name, field.dataType)) ===
1242-
Seq(
1243-
("info_name", StringType),
1244-
("info_value", StringType)
1245-
))
1246-
val description = descriptionDf.collect()
1247-
assert(description === Seq(
1248-
Row("Namespace Name", "ns2"),
1249-
Row(SupportsNamespaces.PROP_COMMENT.capitalize, "test namespace"),
1250-
Row(SupportsNamespaces.PROP_LOCATION.capitalize, "/tmp/ns_test"),
1251-
Row(SupportsNamespaces.PROP_OWNER.capitalize, defaultUser))
1252-
)
1253-
}
1254-
}
1255-
12561236
test("ALTER NAMESPACE .. SET PROPERTIES using v2 catalog") {
12571237
withNamespace("testcat.ns1.ns2") {
12581238
sql("CREATE NAMESPACE IF NOT EXISTS testcat.ns1.ns2 COMMENT " +
@@ -2810,25 +2790,6 @@ class DataSourceV2SQLSuite
28102790
}
28112791
}
28122792

2813-
test("SPARK-34577: drop/add columns to a dataset of `DESCRIBE NAMESPACE`") {
2814-
withNamespace("ns") {
2815-
sql("CREATE NAMESPACE ns")
2816-
val description = sql(s"DESCRIBE NAMESPACE ns")
2817-
val noCommentDataset = description.drop("info_name")
2818-
val expectedSchema = new StructType()
2819-
.add(
2820-
name = "info_value",
2821-
dataType = StringType,
2822-
nullable = true,
2823-
metadata = new MetadataBuilder()
2824-
.putString("comment", "value of the namespace info").build())
2825-
assert(noCommentDataset.schema === expectedSchema)
2826-
val isNullDataset = noCommentDataset
2827-
.withColumn("is_null", noCommentDataset("info_value").isNull)
2828-
assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false))
2829-
}
2830-
}
2831-
28322793
test("SPARK-34923: do not propagate metadata columns through Project") {
28332794
val t1 = s"${catalogAndNamespace}table"
28342795
withTable(t1) {

‎sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala‎

Lines changed: 24 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,29 @@ class InMemoryCatalogedDDLSuite extends DDLSuite with SharedSparkSession {
225225
}
226226
}
227227

228-
abstract class DDLSuite extends QueryTest with SQLTestUtils {
228+
trait CommandTestUtils extends SQLTestUtils {
229+
private val escapedIdentifier = "`(.+)`".r
230+
231+
/**
232+
* Strip backticks, if any, from the string.
233+
*/
234+
protected def cleanIdentifier(ident: String): String = {
235+
ident match {
236+
case escapedIdentifier(i) => i
237+
case plainIdent => plainIdent
238+
}
239+
}
240+
241+
/**
242+
* Returns the path to the database of the given name.
243+
*/
244+
protected def getDBPath(dbName: String): URI = {
245+
val warehousePath = makeQualifiedPath(spark.sessionState.conf.warehousePath)
246+
new Path(CatalogUtils.URIToString(warehousePath), s"$dbName.db").toUri
247+
}
248+
}
249+
250+
abstract class DDLSuite extends QueryTest with CommandTestUtils {
229251

230252
protected val reversedProperties = Seq(PROP_OWNER)
231253

@@ -239,8 +261,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
239261
isDataSource: Boolean = true,
240262
partitionCols: Seq[String] = Seq("a", "b")): CatalogTable
241263

242-
private val escapedIdentifier = "`(.+)`".r
243-
244264
private def dataSource: String = {
245265
if (isUsingHiveMetastore) {
246266
"HIVE"
@@ -258,16 +278,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
258278
assert(normalizeCatalogTable(actual) == normalizeCatalogTable(expected))
259279
}
260280

261-
/**
262-
* Strip backticks, if any, from the string.
263-
*/
264-
private def cleanIdentifier(ident: String): String = {
265-
ident match {
266-
case escapedIdentifier(i) => i
267-
case plainIdent => plainIdent
268-
}
269-
}
270-
271281
private def assertUnsupported(query: String): Unit = {
272282
val e = intercept[AnalysisException] {
273283
sql(query)
@@ -304,11 +314,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
304314
catalog.createPartitions(tableName, Seq(part), ignoreIfExists = false)
305315
}
306316

307-
private def getDBPath(dbName: String): URI = {
308-
val warehousePath = makeQualifiedPath(spark.sessionState.conf.warehousePath)
309-
new Path(CatalogUtils.URIToString(warehousePath), s"$dbName.db").toUri
310-
}
311-
312317
test("alter table: set location (datasource table)") {
313318
testSetLocation(isDatasourceTable = true)
314319
}
@@ -757,14 +762,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
757762

758763
sql(s"CREATE DATABASE $dbName")
759764

760-
checkAnswer(
761-
sql(s"DESCRIBE DATABASE EXTENDED $dbName").toDF("key", "value")
762-
.where("key not like 'Owner%'"), // filter for consistency with in-memory catalog
763-
Row("Database Name", dbNameWithoutBackTicks) ::
764-
Row("Comment", "") ::
765-
Row("Location", CatalogUtils.URIToString(location)) ::
766-
Row("Properties", "") :: Nil)
767-
768765
sql(s"ALTER DATABASE $dbName SET DBPROPERTIES ('a'='a', 'b'='b', 'c'='c')")
769766

770767
checkAnswer(
@@ -813,7 +810,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
813810
}
814811
}
815812

816-
test("Drop/Alter/Describe Database - database does not exists") {
813+
test("Drop/Alter Database - database does not exists") {
817814
val databaseNames = Seq("db1", "`database`")
818815

819816
databaseNames.foreach { dbName =>
@@ -829,13 +826,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
829826
sql(s"ALTER DATABASE $dbName SET DBPROPERTIES ('d'='d')")
830827
}.getMessage
831828
assert(message.contains(s"Database '$dbNameWithoutBackTicks' not found"))
832-
833-
message = intercept[AnalysisException] {
834-
sql(s"DESCRIBE DATABASE EXTENDED $dbName")
835-
}.getMessage
836-
assert(message.contains(s"Database '$dbNameWithoutBackTicks' not found"))
837-
838-
sql(s"DROP DATABASE IF EXISTS $dbName")
839829
}
840830
}
841831

‎sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.sql.execution.command
19+
20+
import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedNamespace}
21+
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
22+
import org.apache.spark.sql.catalyst.plans.logical.DescribeNamespace
23+
24+
class DescribeNamespaceParserSuite extends AnalysisTest {
25+
test("describe namespace") {
26+
val sql1 = "DESCRIBE NAMESPACE EXTENDED a.b"
27+
val sql2 = "DESCRIBE NAMESPACE a.b"
28+
comparePlans(parsePlan(sql1),
29+
DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = true))
30+
comparePlans(parsePlan(sql2),
31+
DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = false))
32+
}
33+
}

‎sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.sql.execution.command
19+
20+
import org.apache.spark.sql.{AnalysisException, QueryTest}
21+
22+
/**
23+
* This base suite contains unified tests for the `DESCRIBE NAMESPACE` command that check V1 and V2
24+
* table catalogs. The tests that cannot run for all supported catalogs are located in more
25+
* specific test suites:
26+
*
27+
* - V2 table catalog tests: `org.apache.spark.sql.execution.command.v2.DescribeNamespaceSuite`
28+
* - V1 table catalog tests:
29+
* `org.apache.spark.sql.execution.command.v1.DescribeNamespaceSuiteBase`
30+
* - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.DescribeNamespaceSuite`
31+
* - V1 Hive External catalog:
32+
* `org.apache.spark.sql.hive.execution.command.DescribeNamespaceSuite`
33+
*/
34+
trait DescribeNamespaceSuiteBase extends QueryTest with DDLCommandTestUtils with CommandTestUtils {
35+
override val command = "DESCRIBE NAMESPACE"
36+
37+
test("namespace does not exists") {
38+
val databaseNames = Seq("db1", "`database`")
39+
databaseNames.foreach { dbName =>
40+
val dbNameWithoutBackTicks = cleanIdentifier(dbName)
41+
val message = intercept[AnalysisException] {
42+
sql(s"DESCRIBE NAMESPACE EXTENDED $catalog.$dbName")
43+
}.getMessage
44+
45+
val prefix = if (catalogVersion == "V2") "Namespace" else "Database"
46+
assert(message.contains(s"$prefix '$dbNameWithoutBackTicks' not found"))
47+
48+
sql(s"DROP NAMESPACE IF EXISTS $catalog.$dbName")
49+
}
50+
}
51+
}

‎sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeNamespaceSuite.scala‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.sql.execution.command.v1
19+
20+
import org.apache.spark.sql.Row
21+
import org.apache.spark.sql.catalyst.catalog.CatalogUtils
22+
import org.apache.spark.sql.execution.command
23+
24+
/**
25+
* This base suite contains unified tests for the `DESCRIBE NAMESPACE` command that checks V1
26+
* table catalogs. The tests that cannot run for all V1 catalogs are located in more
27+
* specific test suites:
28+
*
29+
* - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.DescribeNamespaceSuite`
30+
* - V1 Hive External catalog:
31+
* `org.apache.spark.sql.hive.execution.command.DescribeNamespaceSuite`
32+
*/
33+
trait DescribeNamespaceSuiteBase extends command.DescribeNamespaceSuiteBase {
34+
test("basic") {
35+
val namespaceNames = Seq("db1", "`database`")
36+
withNamespace(namespaceNames: _*) {
37+
namespaceNames.foreach { ns =>
38+
val dbNameWithoutBackTicks = cleanIdentifier(ns)
39+
val location = getDBPath(dbNameWithoutBackTicks)
40+
41+
sql(s"CREATE NAMESPACE $ns")
42+
43+
checkAnswer(
44+
sql(s"DESCRIBE NAMESPACE EXTENDED $ns")
45+
.toDF("key", "value")
46+
.where("key not like 'Owner%'"), // filter for consistency with in-memory catalog
47+
Row("Database Name", dbNameWithoutBackTicks) ::
48+
Row("Comment", "") ::
49+
Row("Location", CatalogUtils.URIToString(location)) ::
50+
Row("Properties", "") :: Nil)
51+
}
52+
}
53+
}
54+
}
55+
56+
/**
57+
* The class contains tests for the `DESCRIBE NAMESPACE` command to check V1 In-Memory
58+
* table catalog.
59+
*/
60+
class DescribeNamespaceSuite extends DescribeNamespaceSuiteBase with CommandSuiteBase

‎sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.sql.execution.command.v2
19+
20+
import org.apache.spark.sql.Row
21+
import org.apache.spark.sql.connector.catalog.SupportsNamespaces
22+
import org.apache.spark.sql.execution.command
23+
import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StringType, StructType}
24+
import org.apache.spark.util.Utils
25+
26+
/**
27+
* The class contains tests for the `DESCRIBE NAMESPACE` command to check V2 table catalogs.
28+
*/
29+
class DescribeNamespaceSuite extends command.DescribeNamespaceSuiteBase with CommandSuiteBase {
30+
test("DescribeNamespace using v2 catalog") {
31+
withNamespace("test_catalog.ns1.ns2") {
32+
sql("CREATE NAMESPACE IF NOT EXISTS test_catalog.ns1.ns2 COMMENT " +
33+
"'test namespace' LOCATION '/tmp/ns_test'")
34+
val descriptionDf = sql("DESCRIBE NAMESPACE test_catalog.ns1.ns2")
35+
assert(descriptionDf.schema.map(field => (field.name, field.dataType)) ===
36+
Seq(
37+
("info_name", StringType),
38+
("info_value", StringType)
39+
))
40+
val description = descriptionDf.collect()
41+
assert(description === Seq(
42+
Row("Namespace Name", "ns2"),
43+
Row(SupportsNamespaces.PROP_COMMENT.capitalize, "test namespace"),
44+
Row(SupportsNamespaces.PROP_LOCATION.capitalize, "/tmp/ns_test"),
45+
Row(SupportsNamespaces.PROP_OWNER.capitalize, Utils.getCurrentUserName()))
46+
)
47+
}
48+
}
49+
50+
test("SPARK-34577: drop/add columns to a dataset of `DESCRIBE NAMESPACE`") {
51+
withNamespace("ns") {
52+
sql("CREATE NAMESPACE ns")
53+
val description = sql(s"DESCRIBE NAMESPACE ns")
54+
val noCommentDataset = description.drop("info_name")
55+
val expectedSchema = new StructType()
56+
.add(
57+
name = "info_value",
58+
dataType = StringType,
59+
nullable = true,
60+
metadata = new MetadataBuilder()
61+
.putString("comment", "value of the namespace info").build())
62+
assert(noCommentDataset.schema === expectedSchema)
63+
val isNullDataset = noCommentDataset
64+
.withColumn("is_null", noCommentDataset("info_value").isNull)
65+
assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false))
66+
}
67+
}
68+
}

0 commit comments

Comments
 (0)