Skip to content

Commit

Permalink
[enhance](mtmv)Improve the speed of obtaining table snapshots (apache…
Browse files Browse the repository at this point in the history
…#40717)

Batch fetch HivePartition to avoid a large number of RPC without cache
  • Loading branch information
zddr authored Sep 19, 2024
1 parent 18048e2 commit 412ff70
Showing 1 changed file with 9 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -760,26 +760,26 @@ public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context) throws Analys
if (getPartitionType() == PartitionType.UNPARTITIONED) {
return new MTMVMaxTimestampSnapshot(getName(), getLastDdlTime());
}
Long maxPartitionId = 0L;
HivePartition maxPartition = null;
long maxVersionTime = 0L;
long visibleVersionTime;
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) getCatalog());
HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues(
getDbName(), getName(), getPartitionColumnTypes());
BiMap<Long, String> idToName = hivePartitionValues.getPartitionNameToIdMap().inverse();
if (MapUtils.isEmpty(idToName)) {
throw new AnalysisException("partitions is empty for : " + getName());
List<HivePartition> partitionList = cache.getAllPartitionsWithCache(getDbName(), getName(),
Lists.newArrayList(hivePartitionValues.getPartitionValuesMap().values()));
if (CollectionUtils.isEmpty(partitionList)) {
throw new AnalysisException("partitionList is empty, table name: " + getName());
}
for (Long partitionId : idToName.keySet()) {
visibleVersionTime = getHivePartitionByIdOrAnalysisException(partitionId, hivePartitionValues,
cache).getLastModifiedTime();
for (HivePartition hivePartition : partitionList) {
visibleVersionTime = hivePartition.getLastModifiedTime();
if (visibleVersionTime > maxVersionTime) {
maxVersionTime = visibleVersionTime;
maxPartitionId = partitionId;
maxPartition = hivePartition;
}
}
return new MTMVMaxTimestampSnapshot(idToName.get(maxPartitionId), maxVersionTime);
return new MTMVMaxTimestampSnapshot(maxPartition.getPartitionName(getPartitionColumns()), maxVersionTime);
}

private Long getPartitionIdByNameOrAnalysisException(String partitionName,
Expand Down

0 comments on commit 412ff70

Please sign in to comment.