From 1af428b68c4341192bf8f66af7c434a7b89be61d Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Wed, 24 Aug 2016 23:26:00 -0700 Subject: [PATCH 1/7] fix --- .../apache/spark/sql/AnalysisException.scala | 3 +++ .../sql/catalyst/catalog/SessionCatalog.scala | 9 +++++-- .../sql/hive/client/HiveClientImpl.scala | 4 +-- .../sql/hive/execution/HiveDDLSuite.scala | 26 +++++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index 691184399939..1247c3744105 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -48,3 +48,6 @@ class AnalysisException protected[sql] ( s"$message;$lineAnnotation$positionAnnotation" } } + +class SQLFeatureNotSupportedException(feature: String) + extends AnalysisException(s"$feature is not supported.") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index 62d0da076b5a..2f91fa2c4f0d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -25,7 +25,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.internal.Logging -import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.{AnalysisException, SQLFeatureNotSupportedException} import org.apache.spark.sql.catalyst.{CatalystConf, SimpleCatalystConf} import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} import org.apache.spark.sql.catalyst.analysis._ @@ -223,6 +223,7 @@ class SessionCatalog( val table = formatTableName(tableDefinition.identifier.table) val newTableDefinition = tableDefinition.copy(identifier = TableIdentifier(table, Some(db))) requireDbExists(db) + requireTableNotExists(TableIdentifier(table, Some(db))) externalCatalog.createTable(newTableDefinition, ignoreIfExists) } @@ -451,7 +452,11 @@ class SessionCatalog( if (isTemporaryTable(name)) { true } else { - externalCatalog.tableExists(db, table) + try { + externalCatalog.tableExists(db, table) + } catch { + case _: SQLFeatureNotSupportedException => true + } } } diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 81d5a124e9d4..6c58fb7fdb9e 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -37,7 +37,7 @@ import org.apache.hadoop.security.UserGroupInformation import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.internal.Logging -import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.{AnalysisException, SQLFeatureNotSupportedException} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException} import org.apache.spark.sql.catalyst.catalog._ @@ -381,7 +381,7 @@ private[hive] class HiveClientImpl( case HiveTableType.MANAGED_TABLE => CatalogTableType.MANAGED case HiveTableType.VIRTUAL_VIEW => CatalogTableType.VIEW case HiveTableType.INDEX_TABLE => - throw new AnalysisException("Hive index table is not supported.") + throw new SQLFeatureNotSupportedException("Hive index table") }, schema = schema, partitionColumnNames = partCols.map(_.name), diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index f00a99b6d0b3..91a9d938051f 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -24,8 +24,10 @@ import org.scalatest.BeforeAndAfterEach import org.apache.spark.internal.config._ import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode} +import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, CatalogTableType} import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.hive.HiveExternalCatalog import org.apache.spark.sql.hive.test.TestHiveSingleton import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SQLTestUtils @@ -637,6 +639,30 @@ class HiveDDLSuite } } + test("create table with the same name as an index table") { + val tabName = "tab1" + val indexName = tabName + "_index" + val indexTabName = "default__tab1_tab1_index__" + withTable(tabName) { + val client = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client + sql(s"CREATE TABLE $tabName(a int)") + + try { + // Spark SQL does not support creating index. Thus, we have to use Hive client. + client.runSqlHive( + s"CREATE INDEX $indexName ON TABLE $tabName (a) AS 'COMPACT' WITH DEFERRED REBUILD") + intercept[TableAlreadyExistsException] { + sql(s"CREATE TABLE $indexTabName(b int)") + } + intercept[TableAlreadyExistsException] { + sql(s"ALTER TABLE $tabName RENAME TO $indexTabName") + } + } finally { + client.runSqlHive(s"DROP INDEX IF EXISTS $indexName ON $tabName") + } + } + } + test("desc table for data source table - no user-defined schema") { Seq("parquet", "json", "orc").foreach { fileFormat => withTable("t1") { From 664d6f1caa9b3d62eafbddb292991def722910ae Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Wed, 24 Aug 2016 23:34:16 -0700 Subject: [PATCH 2/7] improve test cases --- .../apache/spark/sql/hive/execution/HiveDDLSuite.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 91a9d938051f..bf021f64f64b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path import org.scalatest.BeforeAndAfterEach import org.apache.spark.internal.config._ -import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode} +import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode, SQLFeatureNotSupportedException} import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, CatalogTableType} import org.apache.spark.sql.catalyst.TableIdentifier @@ -644,11 +644,11 @@ class HiveDDLSuite val indexName = tabName + "_index" val indexTabName = "default__tab1_tab1_index__" withTable(tabName) { + // Spark SQL does not support creating index. Thus, we have to use Hive client. val client = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client sql(s"CREATE TABLE $tabName(a int)") try { - // Spark SQL does not support creating index. Thus, we have to use Hive client. client.runSqlHive( s"CREATE INDEX $indexName ON TABLE $tabName (a) AS 'COMPACT' WITH DEFERRED REBUILD") intercept[TableAlreadyExistsException] { @@ -657,6 +657,12 @@ class HiveDDLSuite intercept[TableAlreadyExistsException] { sql(s"ALTER TABLE $tabName RENAME TO $indexTabName") } + + // When tableExists is not invoked, we still can get a SQLFeatureNotSupportedException + val e = intercept[SQLFeatureNotSupportedException] { + sql(s"DESCRIBE $indexTabName") + }.getMessage + assert(e.contains("Hive index table is not supported.")) } finally { client.runSqlHive(s"DROP INDEX IF EXISTS $indexName ON $tabName") } From c400c5292a32549cea80861adfaefeb41f4d90b3 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Thu, 25 Aug 2016 00:12:57 -0700 Subject: [PATCH 3/7] fix --- .../src/main/scala/org/apache/spark/sql/AnalysisException.scala | 2 +- .../org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index 1247c3744105..a4fdae4dc396 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -49,5 +49,5 @@ class AnalysisException protected[sql] ( } } -class SQLFeatureNotSupportedException(feature: String) +class SQLFeatureNotSupportedException(val feature: String) extends AnalysisException(s"$feature is not supported.") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index 2f91fa2c4f0d..de351578e3f0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -455,7 +455,7 @@ class SessionCatalog( try { externalCatalog.tableExists(db, table) } catch { - case _: SQLFeatureNotSupportedException => true + case e: SQLFeatureNotSupportedException if e.feature == "Hive index table" => true } } } From 3f7560517955fae5b47d093567690e30988a1925 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Thu, 25 Aug 2016 21:53:41 -0700 Subject: [PATCH 4/7] fix --- .../org/apache/spark/sql/AnalysisException.scala | 3 --- .../spark/sql/catalyst/catalog/SessionCatalog.scala | 9 ++------- .../sql/catalyst/catalog/ExternalCatalogSuite.scala | 10 ++++++++++ .../apache/spark/sql/hive/HiveExternalCatalog.scala | 11 ++++++++++- .../apache/spark/sql/hive/client/HiveClientImpl.scala | 4 ++-- .../spark/sql/hive/execution/HiveDDLSuite.scala | 4 ++-- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index a4fdae4dc396..691184399939 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -48,6 +48,3 @@ class AnalysisException protected[sql] ( s"$message;$lineAnnotation$positionAnnotation" } } - -class SQLFeatureNotSupportedException(val feature: String) - extends AnalysisException(s"$feature is not supported.") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index de351578e3f0..62d0da076b5a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -25,7 +25,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.internal.Logging -import org.apache.spark.sql.{AnalysisException, SQLFeatureNotSupportedException} +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{CatalystConf, SimpleCatalystConf} import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} import org.apache.spark.sql.catalyst.analysis._ @@ -223,7 +223,6 @@ class SessionCatalog( val table = formatTableName(tableDefinition.identifier.table) val newTableDefinition = tableDefinition.copy(identifier = TableIdentifier(table, Some(db))) requireDbExists(db) - requireTableNotExists(TableIdentifier(table, Some(db))) externalCatalog.createTable(newTableDefinition, ignoreIfExists) } @@ -452,11 +451,7 @@ class SessionCatalog( if (isTemporaryTable(name)) { true } else { - try { - externalCatalog.tableExists(db, table) - } catch { - case e: SQLFeatureNotSupportedException if e.feature == "Hive index table" => true - } + externalCatalog.tableExists(db, table) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala index 54365fd978ab..19f866538331 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala @@ -25,6 +25,7 @@ import org.scalatest.BeforeAndAfterEach import org.apache.spark.SparkFunSuite import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} +import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils @@ -162,6 +163,15 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac assert(actual.tableType === CatalogTableType.EXTERNAL) } + test("create table when the table already exists") { + val catalog = newBasicCatalog() + assert(catalog.listTables("db2").toSet == Set("tbl1", "tbl2")) + val table = newTable("tbl1", "db2") + intercept[TableAlreadyExistsException] { + catalog.createTable(table, ignoreIfExists = false) + } + } + test("drop table") { val catalog = newBasicCatalog() assert(catalog.listTables("db2").toSet == Set("tbl1", "tbl2")) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 2586d11a6c1f..1a0e52537210 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -30,6 +30,7 @@ import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.execution.command.DDLUtils import org.apache.spark.sql.execution.datasources.CaseInsensitiveMap @@ -171,9 +172,13 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat ignoreIfExists: Boolean): Unit = withClient { assert(tableDefinition.identifier.database.isDefined) val db = tableDefinition.identifier.database.get + val table = tableDefinition.identifier.table requireDbExists(db) verifyTableProperties(tableDefinition) + if (tableExists(db, table) && !ignoreIfExists) { + throw new TableAlreadyExistsException(db = db, table = table) + } // Before saving data source table metadata into Hive metastore, we should: // 1. Put table schema, partition column names and bucket specification in table properties. // 2. Check if this table is hive compatible @@ -450,7 +455,11 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat } override def tableExists(db: String, table: String): Boolean = withClient { - client.getTableOption(db, table).isDefined + try { + client.getTableOption(db, table).isDefined + } catch { + case e: AnalysisException if e.message.contains("Hive index table is not supported") => true + } } override def listTables(db: String): Seq[String] = withClient { diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 6c58fb7fdb9e..81d5a124e9d4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -37,7 +37,7 @@ import org.apache.hadoop.security.UserGroupInformation import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.internal.Logging -import org.apache.spark.sql.{AnalysisException, SQLFeatureNotSupportedException} +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException} import org.apache.spark.sql.catalyst.catalog._ @@ -381,7 +381,7 @@ private[hive] class HiveClientImpl( case HiveTableType.MANAGED_TABLE => CatalogTableType.MANAGED case HiveTableType.VIRTUAL_VIEW => CatalogTableType.VIEW case HiveTableType.INDEX_TABLE => - throw new SQLFeatureNotSupportedException("Hive index table") + throw new AnalysisException("Hive index table is not supported.") }, schema = schema, partitionColumnNames = partCols.map(_.name), diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index bf021f64f64b..ebb784ff4e5d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path import org.scalatest.BeforeAndAfterEach import org.apache.spark.internal.config._ -import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode, SQLFeatureNotSupportedException} +import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode} import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, CatalogTableType} import org.apache.spark.sql.catalyst.TableIdentifier @@ -659,7 +659,7 @@ class HiveDDLSuite } // When tableExists is not invoked, we still can get a SQLFeatureNotSupportedException - val e = intercept[SQLFeatureNotSupportedException] { + val e = intercept[AnalysisException] { sql(s"DESCRIBE $indexTabName") }.getMessage assert(e.contains("Hive index table is not supported.")) From 8bcd946ac37a36726a8059f2d074551357d2ed2b Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Thu, 25 Aug 2016 21:58:44 -0700 Subject: [PATCH 5/7] update the comment --- .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index ebb784ff4e5d..4dfadcc4f551 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -658,7 +658,7 @@ class HiveDDLSuite sql(s"ALTER TABLE $tabName RENAME TO $indexTabName") } - // When tableExists is not invoked, we still can get a SQLFeatureNotSupportedException + // When tableExists is not invoked, we still can get an AnalysisException val e = intercept[AnalysisException] { sql(s"DESCRIBE $indexTabName") }.getMessage From ce35aa01894e83761226db38d3ca0ce720de5284 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 29 Aug 2016 14:20:03 -0700 Subject: [PATCH 6/7] add tableExists in HiveClient --- .../org/apache/spark/sql/hive/HiveExternalCatalog.scala | 6 +----- .../scala/org/apache/spark/sql/hive/client/HiveClient.scala | 3 +++ .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 4 ++++ .../org/apache/spark/sql/hive/client/VersionsSuite.scala | 6 ++++++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index c8b1d851f304..ed87ac3c3e63 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -455,11 +455,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat } override def tableExists(db: String, table: String): Boolean = withClient { - try { - client.getTableOption(db, table).isDefined - } catch { - case e: AnalysisException if e.message.contains("Hive index table is not supported") => true - } + client.tableExists(db, table) } override def listTables(db: String): Seq[String] = withClient { diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala index 6f009d714bef..dc74fa257aa4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala @@ -68,6 +68,9 @@ private[hive] trait HiveClient { /** List the names of all the databases that match the specified pattern. */ def listDatabases(pattern: String): Seq[String] + /** Return whether a table/view with the specified name exists. */ + def tableExists(dbName: String, tableName: String): Boolean + /** Returns the specified table, or throws [[NoSuchTableException]]. */ final def getTable(dbName: String, tableName: String): CatalogTable = { getTableOption(dbName, tableName).getOrElse(throw new NoSuchTableException(dbName, tableName)) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 81d5a124e9d4..9c17d59c7407 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -347,6 +347,10 @@ private[hive] class HiveClientImpl( client.getDatabasesByPattern(pattern).asScala } + override def tableExists(dbName: String, tableName: String): Boolean = withHiveState { + Option(client.getTable(dbName, tableName, false /* do not throw exception */)).nonEmpty + } + override def getTableOption( dbName: String, tableName: String): Option[CatalogTable] = withHiveState { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala index a2509f2a75f4..10b6cd102416 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala @@ -218,6 +218,12 @@ class VersionsSuite extends SparkFunSuite with Logging { holdDDLTime = false) } + test(s"$version: tableExists") { + // No exception should be thrown + assert(client.tableExists("default", "src")) + assert(!client.tableExists("default", "nonexistent")) + } + test(s"$version: getTable") { // No exception should be thrown client.getTable("default", "src") From 439db0bb86179d376874c83e89f7cf3981fb1b6a Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 29 Aug 2016 19:28:19 -0700 Subject: [PATCH 7/7] address comments. --- .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 4dfadcc4f551..56cefecaa64d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -642,7 +642,6 @@ class HiveDDLSuite test("create table with the same name as an index table") { val tabName = "tab1" val indexName = tabName + "_index" - val indexTabName = "default__tab1_tab1_index__" withTable(tabName) { // Spark SQL does not support creating index. Thus, we have to use Hive client. val client = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client @@ -651,6 +650,8 @@ class HiveDDLSuite try { client.runSqlHive( s"CREATE INDEX $indexName ON TABLE $tabName (a) AS 'COMPACT' WITH DEFERRED REBUILD") + val indexTabName = + spark.sessionState.catalog.listTables("default", s"*$indexName*").head.table intercept[TableAlreadyExistsException] { sql(s"CREATE TABLE $indexTabName(b int)") }