From d2a68b36ab536808271d269c7dbe99f5a2434d5c Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Tue, 30 Jul 2019 09:57:19 +0800 Subject: [PATCH 01/13] init pr --- .../spark/sql/catalyst/parser/SqlBase.g4 | 2 ++ .../spark/sql/execution/SparkSqlParser.scala | 16 ++++++++++++++ .../spark/sql/execution/command/ddl.scala | 21 +++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 517ef9de4964..e97b81f8f2d7 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -89,6 +89,8 @@ statement (WITH DBPROPERTIES tablePropertyList))* #createDatabase | ALTER database db=errorCapturingIdentifier SET DBPROPERTIES tablePropertyList #setDatabaseProperties + | ALTER database db=errorCapturingIdentifier + SET locationSpec #setDatabaseLocation | DROP database (IF EXISTS)? db=errorCapturingIdentifier (RESTRICT | CASCADE)? #dropDatabase | SHOW DATABASES (LIKE? pattern=STRING)? #showDatabases diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 12cd8abcad89..f3864c32a213 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -532,6 +532,22 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { visitPropertyKeyValues(ctx.tablePropertyList)) } + /** + * Create an [[AlterDatabaseLocationCommand]] command. + * + * For example: + * {{{ + * ALTER (DATABASE|SCHEMA) database SET LOCATION path; + * }}} + */ + override def visitSetDatabaseLocation( + ctx: SetDatabaseLocationContext): LogicalPlan = withOrigin(ctx) { + AlterDatabaseLocationCommand( + ctx.db.getText, + visitLocationSpec(ctx.locationSpec()) + ) + } + /** * Create a [[DropDatabaseCommand]] command. * diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index ee5d37cebf2f..7429d19523e4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -132,6 +132,27 @@ case class AlterDatabasePropertiesCommand( } } +/** + * A command for users to set new location path for a database + * If the database does not exist, an error message will be issued to indicate the database + * does not exist. + * The syntax of using this command in SQL is: + * {{{ + * ALTER (DATABASE|SCHEMA) database_name SET LOCATION path + * }}} + */ +case class AlterDatabaseLocationCommand(databaseName: String, location: String) + extends RunnableCommand { + + override def run(sparkSession: SparkSession): Seq[Row] = { + val catalog = sparkSession.sessionState.catalog + val db: CatalogDatabase = catalog.getDatabaseMetadata(databaseName) + catalog.alterDatabase(db.copy(locationUri = CatalogUtils.stringToURI(location))) + + Seq.empty[Row] + } +} + /** * A command for users to show the name of the database, its comment (if one has been set), and its * root location on the filesystem. When extended is true, it also shows the database's properties From 71033f8cdcb2d82dc6d4689e28044aff367a0e88 Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Tue, 30 Jul 2019 22:12:35 +0800 Subject: [PATCH 02/13] add test --- .../spark/sql/execution/command/DDLParserSuite.scala | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala index 83452cdd8927..5889572838b1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala @@ -184,6 +184,15 @@ class DDLParserSuite extends AnalysisTest with SharedSQLContext { containsThesePhrases = Seq("key_without_value")) } + test("alter database set location") { + // ALTER (DATABASE|SCHEMA) database_name SET LOCATION + val sql1 = "ALTER DATABASE database_name SET LOCATION '/home/user/db'" + val parsed1 = parser.parsePlan(sql1) + + val expected1 = AlterDatabaseLocationCommand("database_name", "/home/user/db") + comparePlans(parsed1, expected1) + } + test("describe database") { // DESCRIBE DATABASE [EXTENDED] db_name; val sql1 = "DESCRIBE DATABASE EXTENDED db_name" From f3eb6e9e911cfda478733be0de32b9597e4e7286 Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Wed, 31 Jul 2019 00:02:46 +0800 Subject: [PATCH 03/13] fix style --- .../apache/spark/sql/execution/SparkSqlParser.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index f3864c32a213..beac0527a671 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -541,11 +541,11 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { * }}} */ override def visitSetDatabaseLocation( - ctx: SetDatabaseLocationContext): LogicalPlan = withOrigin(ctx) { - AlterDatabaseLocationCommand( - ctx.db.getText, - visitLocationSpec(ctx.locationSpec()) - ) + ctx: SetDatabaseLocationContext): LogicalPlan = withOrigin(ctx) { + AlterDatabaseLocationCommand( + ctx.db.getText, + visitLocationSpec(ctx.locationSpec()) + ) } /** From bd4fb39e9ce3fdb8b80dd7eafefa3828f4381fac Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Wed, 31 Jul 2019 09:34:52 +0800 Subject: [PATCH 04/13] address comments --- .../apache/spark/sql/execution/SparkSqlParser.scala | 6 +++--- .../org/apache/spark/sql/execution/command/ddl.scala | 2 +- .../spark/sql/execution/command/DDLParserSuite.scala | 2 +- .../apache/spark/sql/execution/command/DDLSuite.scala | 10 ++++++++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index beac0527a671..125fee62ad80 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -533,7 +533,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { } /** - * Create an [[AlterDatabaseLocationCommand]] command. + * Create an [[AlterDatabaseSetLocationCommand]] command. * * For example: * {{{ @@ -542,9 +542,9 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { */ override def visitSetDatabaseLocation( ctx: SetDatabaseLocationContext): LogicalPlan = withOrigin(ctx) { - AlterDatabaseLocationCommand( + AlterDatabaseSetLocationCommand( ctx.db.getText, - visitLocationSpec(ctx.locationSpec()) + visitLocationSpec(ctx.locationSpec) ) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 7429d19523e4..c291cda00216 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -141,7 +141,7 @@ case class AlterDatabasePropertiesCommand( * ALTER (DATABASE|SCHEMA) database_name SET LOCATION path * }}} */ -case class AlterDatabaseLocationCommand(databaseName: String, location: String) +case class AlterDatabaseSetLocationCommand(databaseName: String, location: String) extends RunnableCommand { override def run(sparkSession: SparkSession): Seq[Row] = { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala index 5889572838b1..8435df95ee6e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala @@ -189,7 +189,7 @@ class DDLParserSuite extends AnalysisTest with SharedSQLContext { val sql1 = "ALTER DATABASE database_name SET LOCATION '/home/user/db'" val parsed1 = parser.parsePlan(sql1) - val expected1 = AlterDatabaseLocationCommand("database_name", "/home/user/db") + val expected1 = AlterDatabaseSetLocationCommand("database_name", "/home/user/db") comparePlans(parsed1, expected1) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index b777db750a1b..613b36adaa7f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -730,6 +730,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { try { val dbNameWithoutBackTicks = cleanIdentifier(dbName) val location = getDBPath(dbNameWithoutBackTicks) + val location2 = getDBPath(dbNameWithoutBackTicks + "2") sql(s"CREATE DATABASE $dbName") @@ -757,6 +758,15 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { Row("Description", "") :: Row("Location", CatalogUtils.URIToString(location)) :: Row("Properties", "((a,a), (b,b), (c,c), (d,d))") :: Nil) + + sql(s"ALTER DATABASE $dbName SET LOCATION '$location2'") + + checkAnswer( + sql(s"DESCRIBE DATABASE EXTENDED $dbName"), + Row("Database Name", dbNameWithoutBackTicks) :: + Row("Description", "") :: + Row("Location", CatalogUtils.URIToString(location2)) :: + Row("Properties", "((a,a), (b,b), (c,c), (d,d))") :: Nil) } finally { catalog.reset() } From 7140624e95010df10def6498b970194fba127319 Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Wed, 31 Jul 2019 17:37:34 +0800 Subject: [PATCH 05/13] fix test --- .../spark/sql/execution/command/DDLSuite.scala | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 613b36adaa7f..2fbf5be06e1b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -759,14 +759,15 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { Row("Location", CatalogUtils.URIToString(location)) :: Row("Properties", "((a,a), (b,b), (c,c), (d,d))") :: Nil) - sql(s"ALTER DATABASE $dbName SET LOCATION '$location2'") - - checkAnswer( - sql(s"DESCRIBE DATABASE EXTENDED $dbName"), - Row("Database Name", dbNameWithoutBackTicks) :: - Row("Description", "") :: - Row("Location", CatalogUtils.URIToString(location2)) :: - Row("Properties", "((a,a), (b,b), (c,c), (d,d))") :: Nil) + withTempDir { tmpDir => + val path = new Path(tmpDir.getCanonicalPath).toUri + val dbNameWithoutBackTicks = cleanIdentifier(dbName) + sql(s"ALTER DATABASE $dbName SET LOCATION '$path'") + val dbPath = makeQualifiedPath( + catalog.getDatabaseMetadata(dbNameWithoutBackTicks).locationUri.toString) + val expPath = makeQualifiedPath(tmpDir.toString) + assert(dbPath === expPath) + } } finally { catalog.reset() } From d209d857ff51c6ff5849edaa97c568393d6ff9ac Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Wed, 31 Jul 2019 23:56:57 +0800 Subject: [PATCH 06/13] fix test --- .../org/apache/spark/sql/execution/command/DDLSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 2fbf5be06e1b..a33d0ae3ff63 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -764,8 +764,8 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { val dbNameWithoutBackTicks = cleanIdentifier(dbName) sql(s"ALTER DATABASE $dbName SET LOCATION '$path'") val dbPath = makeQualifiedPath( - catalog.getDatabaseMetadata(dbNameWithoutBackTicks).locationUri.toString) - val expPath = makeQualifiedPath(tmpDir.toString) + catalog.getDatabaseMetadata(dbNameWithoutBackTicks).locationUri.toString).getPath + val expPath = makeQualifiedPath(tmpDir.toString).getPath assert(dbPath === expPath) } } finally { From 93d76d68c493074e568cae1661d0b314a1948f4a Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Thu, 1 Aug 2019 09:27:52 +0800 Subject: [PATCH 07/13] fix test --- .../spark/sql/execution/command/DDLSuite.scala | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index a33d0ae3ff63..77f8c2602b37 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -760,13 +760,14 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { Row("Properties", "((a,a), (b,b), (c,c), (d,d))") :: Nil) withTempDir { tmpDir => - val path = new Path(tmpDir.getCanonicalPath).toUri - val dbNameWithoutBackTicks = cleanIdentifier(dbName) - sql(s"ALTER DATABASE $dbName SET LOCATION '$path'") - val dbPath = makeQualifiedPath( - catalog.getDatabaseMetadata(dbNameWithoutBackTicks).locationUri.toString).getPath - val expPath = makeQualifiedPath(tmpDir.toString).getPath - assert(dbPath === expPath) + val path = tmpDir.getCanonicalPath + val uri = tmpDir.toURI + sql(s"ALTER DATABASE $dbName SET LOCATION '$uri'") + val pathInCatalog = new Path( + catalog.getDatabaseMetadata(dbNameWithoutBackTicks).locationUri).toUri + assert("file" === pathInCatalog.getScheme) + val expectedPath = new Path(path).toUri + assert(expectedPath.getPath === pathInCatalog.getPath) } } finally { catalog.reset() From 11255a5d17846a9a3cbf1e07e5a9534b5d0ce00c Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Fri, 2 Aug 2019 16:59:56 +0800 Subject: [PATCH 08/13] add test in hive VersionsSuite --- .../apache/spark/sql/hive/client/VersionsSuite.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala index feb364ec1947..9786f4b98f3a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala @@ -21,12 +21,13 @@ import java.io.{ByteArrayOutputStream, File, PrintStream, PrintWriter} import java.net.URI import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path import org.apache.hadoop.hive.common.StatsSetupConst import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe import org.apache.hadoop.mapred.TextInputFormat - import org.apache.spark.SparkFunSuite + import org.apache.spark.internal.Logging import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} @@ -156,6 +157,7 @@ class VersionsSuite extends SparkFunSuite with Logging { /////////////////////////////////////////////////////////////////////////// val tempDatabasePath = Utils.createTempDir().toURI + val tempDatabasePath2 = Utils.createTempDir().toURI test(s"$version: createDatabase") { val defaultDB = CatalogDatabase("default", "desc", new URI("loc"), Map()) @@ -197,6 +199,12 @@ class VersionsSuite extends SparkFunSuite with Logging { val database = client.getDatabase("temporary").copy(properties = Map("flag" -> "true")) client.alterDatabase(database) assert(client.getDatabase("temporary").properties.contains("flag")) + + // test alter database location + client.alterDatabase(database.copy(locationUri = tempDatabasePath2)) + val pathInCatalog = new Path(client.getDatabase("temporary").locationUri).toUri + val expectedPath = new Path(tempDatabasePath2).toUri + assert(expectedPath.getPath === pathInCatalog.getPath) } test(s"$version: dropDatabase") { From 063195954370e34df29bd13ff7a2227f8576c963 Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Fri, 2 Aug 2019 18:48:09 +0800 Subject: [PATCH 09/13] fix style --- .../scala/org/apache/spark/sql/hive/client/VersionsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala index 9786f4b98f3a..e2e390f58936 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala @@ -26,8 +26,8 @@ import org.apache.hadoop.hive.common.StatsSetupConst import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe import org.apache.hadoop.mapred.TextInputFormat -import org.apache.spark.SparkFunSuite +import org.apache.spark.SparkFunSuite import org.apache.spark.internal.Logging import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} From c51f31c8d253032d5f2fc045cd9ad2b1881926c3 Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Sat, 3 Aug 2019 00:12:29 +0800 Subject: [PATCH 10/13] update --- .../org/apache/spark/sql/hive/HiveExternalCatalog.scala | 6 ++++-- .../apache/spark/sql/hive/client/HiveClientImpl.scala | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index d4df35c8ec69..4d081c698939 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -205,12 +205,14 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat */ override def alterDatabase(dbDefinition: CatalogDatabase): Unit = withClient { val existingDb = getDatabase(dbDefinition.name) - if (existingDb.properties == dbDefinition.properties) { + if (existingDb.properties == dbDefinition.properties && + existingDb.locationUri == dbDefinition.locationUri) { logWarning(s"Request to alter database ${dbDefinition.name} is a no-op because " + s"the provided database properties are the same as the old ones. Hive does not " + - s"currently support altering other database fields.") + s"currently support altering other database fields and database location.") } client.alterDatabase(dbDefinition) + client } override def getDatabase(db: String): CatalogDatabase = withClient { diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index fc05d9338525..7629822450ea 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -361,13 +361,16 @@ private[hive] class HiveClientImpl( } override def alterDatabase(database: CatalogDatabase): Unit = withHiveState { + val location = CatalogUtils.URIToString(database.locationUri) + val dbName = database.name client.alterDatabase( - database.name, + dbName, new HiveDatabase( - database.name, + dbName, database.description, - CatalogUtils.URIToString(database.locationUri), + location, Option(database.properties).map(_.asJava).orNull)) + client.getDatabase(dbName).setLocationUri(location) } override def getDatabase(dbName: String): CatalogDatabase = withHiveState { From 734c302a534669e032f3dfbe3fa2b938b6fa11ad Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Sat, 3 Aug 2019 00:57:13 +0800 Subject: [PATCH 11/13] update --- .../spark/sql/hive/client/HiveClientImpl.scala | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 7629822450ea..1688e63bdb7d 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -361,16 +361,11 @@ private[hive] class HiveClientImpl( } override def alterDatabase(database: CatalogDatabase): Unit = withHiveState { - val location = CatalogUtils.URIToString(database.locationUri) - val dbName = database.name - client.alterDatabase( - dbName, - new HiveDatabase( - dbName, - database.description, - location, - Option(database.properties).map(_.asJava).orNull)) - client.getDatabase(dbName).setLocationUri(location) + val hiveDb = client.getDatabase(database.name) + hiveDb.setDescription(database.description) + hiveDb.setLocationUri(CatalogUtils.URIToString(database.locationUri)) + hiveDb.setParameters(Option(database.properties).map(_.asJava).orNull) + client.alterDatabase(database.name, hiveDb) } override def getDatabase(dbName: String): CatalogDatabase = withHiveState { From efed7d02966ad1a62c2d7ad279e4eb8b1a005cde Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Sat, 3 Aug 2019 12:17:30 +0800 Subject: [PATCH 12/13] add log for debug --- .../org/apache/spark/sql/execution/command/DDLSuite.scala | 5 ++++- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 77f8c2602b37..f265e4baf9f6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -759,9 +759,12 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { Row("Location", CatalogUtils.URIToString(location)) :: Row("Properties", "((a,a), (b,b), (c,c), (d,d))") :: Nil) - withTempDir { tmpDir => + withTempDir { _tmpDir => + val tmpDir = new File(_tmpDir, "db1.db") val path = tmpDir.getCanonicalPath val uri = tmpDir.toURI + logWarning(s"test change location: oldPath=${CatalogUtils.URIToString(location)}," + + s" newPath=${CatalogUtils.URIToString(uri)}") sql(s"ALTER DATABASE $dbName SET LOCATION '$uri'") val pathInCatalog = new Path( catalog.getDatabaseMetadata(dbNameWithoutBackTicks).locationUri).toUri diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 1688e63bdb7d..5076a7a9dea2 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -362,10 +362,15 @@ private[hive] class HiveClientImpl( override def alterDatabase(database: CatalogDatabase): Unit = withHiveState { val hiveDb = client.getDatabase(database.name) + val oldPath = hiveDb.getLocationUri + val newPath = CatalogUtils.URIToString(database.locationUri) hiveDb.setDescription(database.description) hiveDb.setLocationUri(CatalogUtils.URIToString(database.locationUri)) hiveDb.setParameters(Option(database.properties).map(_.asJava).orNull) client.alterDatabase(database.name, hiveDb) + val newPathInHive = client.getDatabase(database.name).getLocationUri + logWarning(s"Change database location, " + + s"oldPath=${oldPath}, newPath=${newPath}, newPathInHive=${newPathInHive}") } override def getDatabase(dbName: String): CatalogDatabase = withHiveState { From 2a95580b985dfcf43e2c4d3336ce0286b7c245c0 Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Sat, 3 Aug 2019 13:37:07 +0800 Subject: [PATCH 13/13] use shim msc --- .../org/apache/spark/sql/execution/command/DDLSuite.scala | 3 +-- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index f265e4baf9f6..55e8a75cda1c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -759,8 +759,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { Row("Location", CatalogUtils.URIToString(location)) :: Row("Properties", "((a,a), (b,b), (c,c), (d,d))") :: Nil) - withTempDir { _tmpDir => - val tmpDir = new File(_tmpDir, "db1.db") + withTempDir { tmpDir => val path = tmpDir.getCanonicalPath val uri = tmpDir.toURI logWarning(s"test change location: oldPath=${CatalogUtils.URIToString(location)}," + diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 5076a7a9dea2..8e08ae45680f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -361,14 +361,14 @@ private[hive] class HiveClientImpl( } override def alterDatabase(database: CatalogDatabase): Unit = withHiveState { - val hiveDb = client.getDatabase(database.name) + val hiveDb = msClient.getDatabase(database.name) val oldPath = hiveDb.getLocationUri val newPath = CatalogUtils.URIToString(database.locationUri) hiveDb.setDescription(database.description) hiveDb.setLocationUri(CatalogUtils.URIToString(database.locationUri)) hiveDb.setParameters(Option(database.properties).map(_.asJava).orNull) - client.alterDatabase(database.name, hiveDb) - val newPathInHive = client.getDatabase(database.name).getLocationUri + msClient.alterDatabase(database.name, hiveDb) + val newPathInHive = msClient.getDatabase(database.name).getLocationUri logWarning(s"Change database location, " + s"oldPath=${oldPath}, newPath=${newPath}, newPathInHive=${newPathInHive}") }