Skip to content

[SPARK-35531][SQL] Update hive table stats without unnecessary convert #38495

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wankunde wants to merge 10 commits into from
Closed

[SPARK-35531][SQL] Update hive table stats without unnecessary convert #38495

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
e763d0a
[SPARK-35531][SQL] Update hive table stats without unecessary convert
wankunde Nov 3, 2022
12e1642
Bug fix
wankunde Nov 4, 2022
41890de
Fix UT
wankunde Nov 4, 2022
395b571
Update code
wankunde Nov 15, 2022
8760da9
Update code
wankunde Nov 15, 2022
be0c869
Update code
wankunde Nov 21, 2022
3115a62
Bug fix
wankunde Nov 22, 2022
ee08735
Add hiveTableProps method in hiveTableProps
wankunde Nov 22, 2022
6ebaa06
Use filterNot method instead of filterKeys method
wankunde Nov 22, 2022
bad2444
Remove containsStats parameter in hiveTableProps method
wankunde Nov 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 6 additions & 9 deletions sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -721,19 +721,16 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
table: String,
stats: Option[CatalogStatistics]): Unit = withClient {
requireTableExists(db, table)
val rawTable = getRawTable(db, table)

// convert table statistics to properties so that we can persist them through hive client
val statsProperties =
val rawHiveTable = client.getRawHiveTable(db, table)
val oldProps = rawHiveTable.hiveTableProps().filterNot(_._1.startsWith(STATISTICS_PREFIX))
val newProps =
if (stats.isDefined) {
statsToProperties(stats.get)
oldProps ++ statsToProperties(stats.get)
} else {
new mutable.HashMap[String, String]()
oldProps
}

val oldTableNonStatsProps = rawTable.properties.filterNot(_._1.startsWith(STATISTICS_PREFIX))
val updatedTable = rawTable.copy(properties = oldTableNonStatsProps ++ statsProperties)
client.alterTable(updatedTable)
client.alterTableProps(rawHiveTable, newProps)
}

override def getTable(db: String, table: String): CatalogTable = withClient {
Expand Down
6 changes: 6 additions & 0 deletions sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ import org.apache.spark.sql.types.StructType
private[hive] trait RawHiveTable {
def rawTable: Object
def toCatalogTable: CatalogTable

/** Get hive table properties. */
def hiveTableProps(): Map[String, String]
}

/**
Expand Down Expand Up @@ -127,6 +130,9 @@ private[hive] trait HiveClient {
*/
def alterTable(dbName: String, tableName: String, table: CatalogTable): Unit

/** Alter a table properties */
def alterTableProps(rawHiveTable: RawHiveTable, newProps: Map[String, String]): Unit

/**
* Updates the given table with a new data schema and table properties, and keep everything else
* unchanged.
Expand Down
16 changes: 15 additions & 1 deletion sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import java.lang.{Iterable => JIterable}
import java.lang.reflect.InvocationTargetException
import java.net.URI
import java.nio.charset.StandardCharsets.UTF_8
import java.util.{Locale, Map => JMap}
import java.util.{HashMap => JHashMap, Locale, Map => JMap}
import java.util.concurrent.TimeUnit._

import scala.collection.JavaConverters._
Expand Down Expand Up @@ -105,6 +105,10 @@ private[hive] class HiveClientImpl(

private class RawHiveTableImpl(override val rawTable: HiveTable) extends RawHiveTable {
override lazy val toCatalogTable = convertHiveTableToCatalogTable(rawTable)

override def hiveTableProps(): Map[String, String] = {
rawTable.getParameters.asScala.toMap
}
}

import HiveClientImpl._
Expand Down Expand Up @@ -609,6 +613,16 @@ private[hive] class HiveClientImpl(
shim.alterTable(client, qualifiedTableName, hiveTable)
}

override def alterTableProps(
rawHiveTable: RawHiveTable,
newProps: Map[String, String]): Unit = withHiveState {
val hiveTable = rawHiveTable.rawTable.asInstanceOf[HiveTable]
val newPropsMap = new JHashMap[String, String]()
newPropsMap.putAll(newProps.asJava)
hiveTable.getTTable.setParameters(newPropsMap)
shim.alterTable(client, s"${hiveTable.getDbName}.${hiveTable.getTableName}", hiveTable)
}

override def alterTableDataSchema(
dbName: String,
tableName: String,
Expand Down