diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 084c7b7210401..9af34ba3b2060 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -721,19 +721,16 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat table: String, stats: Option[CatalogStatistics]): Unit = withClient { requireTableExists(db, table) - val rawTable = getRawTable(db, table) - - // convert table statistics to properties so that we can persist them through hive client - val statsProperties = + val rawHiveTable = client.getRawHiveTable(db, table) + val oldProps = rawHiveTable.hiveTableProps().filterNot(_._1.startsWith(STATISTICS_PREFIX)) + val newProps = if (stats.isDefined) { - statsToProperties(stats.get) + oldProps ++ statsToProperties(stats.get) } else { - new mutable.HashMap[String, String]() + oldProps } - val oldTableNonStatsProps = rawTable.properties.filterNot(_._1.startsWith(STATISTICS_PREFIX)) - val updatedTable = rawTable.copy(properties = oldTableNonStatsProps ++ statsProperties) - client.alterTable(updatedTable) + client.alterTableProps(rawHiveTable, newProps) } override def getTable(db: String, table: String): CatalogTable = withClient { diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala index 58cacfa1d5ded..ff225e7a50f0d 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala @@ -29,6 +29,9 @@ import org.apache.spark.sql.types.StructType private[hive] trait RawHiveTable { def rawTable: Object def toCatalogTable: CatalogTable + + /** Get hive table properties. */ + def hiveTableProps(): Map[String, String] } /** @@ -127,6 +130,9 @@ private[hive] trait HiveClient { */ def alterTable(dbName: String, tableName: String, table: CatalogTable): Unit + /** Alter a table properties */ + def alterTableProps(rawHiveTable: RawHiveTable, newProps: Map[String, String]): Unit + /** * Updates the given table with a new data schema and table properties, and keep everything else * unchanged. diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 213d930653dd0..d5cba1b978952 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -22,7 +22,7 @@ import java.lang.{Iterable => JIterable} import java.lang.reflect.InvocationTargetException import java.net.URI import java.nio.charset.StandardCharsets.UTF_8 -import java.util.{Locale, Map => JMap} +import java.util.{HashMap => JHashMap, Locale, Map => JMap} import java.util.concurrent.TimeUnit._ import scala.collection.JavaConverters._ @@ -105,6 +105,10 @@ private[hive] class HiveClientImpl( private class RawHiveTableImpl(override val rawTable: HiveTable) extends RawHiveTable { override lazy val toCatalogTable = convertHiveTableToCatalogTable(rawTable) + + override def hiveTableProps(): Map[String, String] = { + rawTable.getParameters.asScala.toMap + } } import HiveClientImpl._ @@ -609,6 +613,16 @@ private[hive] class HiveClientImpl( shim.alterTable(client, qualifiedTableName, hiveTable) } + override def alterTableProps( + rawHiveTable: RawHiveTable, + newProps: Map[String, String]): Unit = withHiveState { + val hiveTable = rawHiveTable.rawTable.asInstanceOf[HiveTable] + val newPropsMap = new JHashMap[String, String]() + newPropsMap.putAll(newProps.asJava) + hiveTable.getTTable.setParameters(newPropsMap) + shim.alterTable(client, s"${hiveTable.getDbName}.${hiveTable.getTableName}", hiveTable) + } + override def alterTableDataSchema( dbName: String, tableName: String,