diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java index c76741130147b0..e839c566036840 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java @@ -24,11 +24,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ConfigBase.DefaultConfHandler; import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.nereids.rules.expression.rules.MultiColumnBound; -import org.apache.doris.nereids.rules.expression.rules.PartitionItemToRange; import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges; -import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges.PartitionItemAndId; -import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges.PartitionItemAndRange; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.qe.ConnectContext; import org.apache.doris.rpc.RpcException; @@ -36,18 +32,14 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Range; import lombok.AllArgsConstructor; import lombok.Data; -import org.apache.hadoop.util.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.lang.reflect.Field; import java.time.Duration; -import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; @@ -123,35 +115,10 @@ private SortedPartitionRanges loadCache( } Map unsortedMap = table.getOriginPartitions(scan); - List> unsortedList = Lists.newArrayList(unsortedMap.entrySet()); - List> sortedRanges = Lists.newArrayListWithCapacity(unsortedMap.size()); - List> defaultPartitions = Lists.newArrayList(); - for (Entry entry : unsortedList) { - PartitionItem partitionItem = entry.getValue(); - Object id = entry.getKey(); - if (!partitionItem.isDefaultPartition()) { - List> ranges = PartitionItemToRange.toRanges(partitionItem); - for (Range range : ranges) { - sortedRanges.add(new PartitionItemAndRange<>(id, partitionItem, range)); - } - } else { - defaultPartitions.add(new PartitionItemAndId<>(id, partitionItem)); - } + SortedPartitionRanges sortedPartitionRanges = SortedPartitionRanges.build(unsortedMap); + if (sortedPartitionRanges == null) { + return null; } - - sortedRanges.sort((o1, o2) -> { - Range span1 = o1.range; - Range span2 = o2.range; - int result = span1.lowerEndpoint().compareTo(span2.lowerEndpoint()); - if (result != 0) { - return result; - } - result = span1.upperEndpoint().compareTo(span2.upperEndpoint()); - return result; - }); - SortedPartitionRanges sortedPartitionRanges = new SortedPartitionRanges( - sortedRanges, defaultPartitions - ); PartitionCacheContext context = new PartitionCacheContext( table.getId(), table.getPartitionMetaVersion(scan), sortedPartitionRanges); partitionCaches.put(key, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 12887cff1d98e7..8015337c06f87c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -33,6 +33,8 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.ExternalSchemaCache.SchemaCacheKey; import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -453,6 +455,18 @@ public boolean supportInternalPartitionPruned() { return false; } + /** + * Get sorted partition ranges for binary search filtering. + * Subclasses can override this method to provide sorted partition ranges + * for efficient partition pruning. + * + * @param scan the catalog relation + * @return sorted partition ranges, or empty if not supported + */ + public Optional> getSortedPartitionRanges(CatalogRelation scan) { + return Optional.empty(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index e7e2125f41d13e..f6c195ad529091 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -55,6 +55,8 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.mtmv.MTMVSnapshotIf; import org.apache.doris.nereids.exceptions.NotSupportedException; +import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.statistics.AnalysisInfo; @@ -360,6 +362,19 @@ public boolean supportInternalPartitionPruned() { return getDlaType() == DLAType.HIVE || getDlaType() == DLAType.HUDI; } + @Override + public Optional> getSortedPartitionRanges(CatalogRelation scan) { + if (getDlaType() != DLAType.HIVE) { + return Optional.empty(); + } + if (CollectionUtils.isEmpty(this.getPartitionColumns())) { + return Optional.empty(); + } + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = getHivePartitionValues( + MvccUtil.getSnapshotFromContext(this)); + return hivePartitionValues.getSortedPartitionRanges(); + } + public SelectedPartitions initHudiSelectedPartitions(Optional tableSnapshot) { if (getDlaType() != DLAType.HUDI) { return SelectedPartitions.NOT_PRUNED; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 883be276f6f327..7180551f64a4f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -53,9 +53,8 @@ import org.apache.doris.metric.Metric; import org.apache.doris.metric.MetricLabel; import org.apache.doris.metric.MetricRepo; -import org.apache.doris.planner.ColumnBound; +import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges; import org.apache.doris.planner.ListPartitionPrunerV2; -import org.apache.doris.planner.PartitionPrunerV2Base.UniqueId; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.LoadingCache; @@ -66,10 +65,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Range; -import com.google.common.collect.RangeMap; import com.google.common.collect.Streams; -import com.google.common.collect.TreeRangeMap; import lombok.Data; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.conf.Configuration; @@ -90,6 +86,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -258,7 +255,6 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { } Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); - Map> idToUniqueIdsMap = Maps.newHashMapWithExpectedSize(partitionNames.size()); for (String partitionName : partitionNames) { long partitionId = Util.genIdByName(catalog.getName(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName(), partitionName); @@ -267,23 +263,8 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { partitionNameToIdMap.put(partitionName, partitionId); } - Map> uidToPartitionRange = null; - Map, UniqueId> rangeToId = null; - RangeMap singleColumnRangeMap = null; - Map> singleUidToColumnRangeMap = null; - if (key.types.size() > 1) { - // uidToPartitionRange and rangeToId are only used for multi-column partition - uidToPartitionRange = ListPartitionPrunerV2.genUidToPartitionRange(idToPartitionItem, idToUniqueIdsMap); - rangeToId = ListPartitionPrunerV2.genRangeToId(uidToPartitionRange); - } else { - Preconditions.checkState(key.types.size() == 1, key.types); - // singleColumnRangeMap is only used for single-column partition - singleColumnRangeMap = ListPartitionPrunerV2.genSingleColumnRangeMap(idToPartitionItem, idToUniqueIdsMap); - singleUidToColumnRangeMap = ListPartitionPrunerV2.genSingleUidToColumnRange(singleColumnRangeMap); - } Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - return new HivePartitionValues(idToPartitionItem, uidToPartitionRange, rangeToId, singleColumnRangeMap, - partitionNameToIdMap, idToUniqueIdsMap, singleUidToColumnRangeMap, partitionValuesMap); + return new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap); } @VisibleForTesting @@ -614,7 +595,6 @@ public void addPartitionsCache(NameMapping nameMapping, List partitionNa HivePartitionValues copy = partitionValues.copy(); Map idToPartitionItemBefore = copy.getIdToPartitionItem(); Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); - Map> idToUniqueIdsMap = copy.getIdToUniqueIdsMap(); Map idToPartitionItem = new HashMap<>(); String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); @@ -632,28 +612,8 @@ public void addPartitionsCache(NameMapping nameMapping, List partitionNa Map> partitionValuesMapBefore = copy.getPartitionValuesMap(); Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); partitionValuesMapBefore.putAll(partitionValuesMap); - if (key.types.size() > 1) { - Map> uidToPartitionRangeBefore = copy.getUidToPartitionRange(); - // uidToPartitionRange and rangeToId are only used for multi-column partition - Map> uidToPartitionRange = ListPartitionPrunerV2 - .genUidToPartitionRange(idToPartitionItem, idToUniqueIdsMap); - uidToPartitionRangeBefore.putAll(uidToPartitionRange); - Map, UniqueId> rangeToIdBefore = copy.getRangeToId(); - Map, UniqueId> rangeToId = ListPartitionPrunerV2.genRangeToId(uidToPartitionRange); - rangeToIdBefore.putAll(rangeToId); - } else { - Preconditions.checkState(key.types.size() == 1, key.types); - // singleColumnRangeMap is only used for single-column partition - RangeMap singleColumnRangeMapBefore = copy.getSingleColumnRangeMap(); - RangeMap singleColumnRangeMap = ListPartitionPrunerV2 - .genSingleColumnRangeMap(idToPartitionItem, idToUniqueIdsMap); - singleColumnRangeMapBefore.putAll(singleColumnRangeMap); - Map> singleUidToColumnRangeMapBefore = copy - .getSingleUidToColumnRangeMap(); - Map> singleUidToColumnRangeMap = ListPartitionPrunerV2 - .genSingleUidToColumnRange(singleColumnRangeMap); - singleUidToColumnRangeMapBefore.putAll(singleUidToColumnRangeMap); - } + // Rebuild sorted partition ranges after adding partitions + copy.rebuildSortedPartitionRanges(); HivePartitionValues partitionValuesCur = partitionValuesCache.getIfPresent(key); if (partitionValuesCur == partitionValues) { partitionValuesCache.put(key, copy); @@ -671,11 +631,6 @@ public void dropPartitionsCache(ExternalTable dorisTable, List partition HivePartitionValues copy = partitionValues.copy(); Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); Map idToPartitionItemBefore = copy.getIdToPartitionItem(); - Map> idToUniqueIdsMapBefore = copy.getIdToUniqueIdsMap(); - Map> uidToPartitionRangeBefore = copy.getUidToPartitionRange(); - Map, UniqueId> rangeToIdBefore = copy.getRangeToId(); - RangeMap singleColumnRangeMapBefore = copy.getSingleColumnRangeMap(); - Map> singleUidToColumnRangeMapBefore = copy.getSingleUidToColumnRangeMap(); Map> partitionValuesMap = copy.getPartitionValuesMap(); for (String partitionName : partitionNames) { if (!partitionNameToIdMapBefore.containsKey(partitionName)) { @@ -686,27 +641,13 @@ public void dropPartitionsCache(ExternalTable dorisTable, List partition Long partitionId = partitionNameToIdMapBefore.remove(partitionName); idToPartitionItemBefore.remove(partitionId); partitionValuesMap.remove(partitionId); - List uniqueIds = idToUniqueIdsMapBefore.remove(partitionId); - for (UniqueId uniqueId : uniqueIds) { - if (uidToPartitionRangeBefore != null) { - Range range = uidToPartitionRangeBefore.remove(uniqueId); - if (range != null) { - rangeToIdBefore.remove(range); - } - } - - if (singleUidToColumnRangeMapBefore != null) { - Range range = singleUidToColumnRangeMapBefore.remove(uniqueId); - if (range != null) { - singleColumnRangeMapBefore.remove(range); - } - } - } if (invalidPartitionCache) { invalidatePartitionCache(dorisTable, partitionName); } } + // Rebuild sorted partition ranges after dropping partitions + copy.rebuildSortedPartitionRanges(); HivePartitionValues partitionValuesCur = partitionValuesCache.getIfPresent(key); if (partitionValuesCur == partitionValues) { partitionValuesCache.put(key, copy); @@ -981,7 +922,7 @@ public boolean equals(Object obj) { return dummyKey == ((FileCacheKey) obj).dummyKey; } return location.equals(((FileCacheKey) obj).location) - && Objects.equals(partitionValues, ((FileCacheKey) obj).partitionValues); + && Objects.equals(partitionValues, ((FileCacheKey) obj).partitionValues); } boolean isSameTable(long id) { @@ -1079,54 +1020,69 @@ public static class HiveFileStatus { @Data public static class HivePartitionValues { private BiMap partitionNameToIdMap; - private Map> idToUniqueIdsMap; private Map idToPartitionItem; private Map> partitionValuesMap; - //multi pair - private Map> uidToPartitionRange; - private Map, UniqueId> rangeToId; - //single pair - private RangeMap singleColumnRangeMap; - private Map> singleUidToColumnRangeMap; + + // Sorted partition ranges for binary search filtering. + // Built at construction time, shares the same lifecycle with HivePartitionValues. + private SortedPartitionRanges sortedPartitionRanges; public HivePartitionValues() { } public HivePartitionValues(Map idToPartitionItem, - Map> uidToPartitionRange, - Map, UniqueId> rangeToId, - RangeMap singleColumnRangeMap, BiMap partitionNameToIdMap, - Map> idToUniqueIdsMap, - Map> singleUidToColumnRangeMap, Map> partitionValuesMap) { this.idToPartitionItem = idToPartitionItem; - this.uidToPartitionRange = uidToPartitionRange; - this.rangeToId = rangeToId; - this.singleColumnRangeMap = singleColumnRangeMap; this.partitionNameToIdMap = partitionNameToIdMap; - this.idToUniqueIdsMap = idToUniqueIdsMap; - this.singleUidToColumnRangeMap = singleUidToColumnRangeMap; this.partitionValuesMap = partitionValuesMap; + this.sortedPartitionRanges = buildSortedPartitionRanges(); } + /** + * Create a copy for incremental updates (add/drop partitions). + * The sortedPartitionRanges will be rebuilt after the caller modifies the partition data. + */ public HivePartitionValues copy() { HivePartitionValues copy = new HivePartitionValues(); copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap)); - copy.setIdToUniqueIdsMap(idToUniqueIdsMap == null ? null : Maps.newHashMap(idToUniqueIdsMap)); copy.setIdToPartitionItem(idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem)); copy.setPartitionValuesMap(partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap)); - copy.setUidToPartitionRange(uidToPartitionRange == null ? null : Maps.newHashMap(uidToPartitionRange)); - copy.setRangeToId(rangeToId == null ? null : Maps.newHashMap(rangeToId)); - copy.setSingleUidToColumnRangeMap( - singleUidToColumnRangeMap == null ? null : Maps.newHashMap(singleUidToColumnRangeMap)); - if (singleColumnRangeMap != null) { - RangeMap copySingleColumnRangeMap = TreeRangeMap.create(); - copySingleColumnRangeMap.putAll(singleColumnRangeMap); - copy.setSingleColumnRangeMap(copySingleColumnRangeMap); - } + // sortedPartitionRanges is not copied here, caller should call rebuildSortedPartitionRanges() + // after modifying partition data return copy; } + + /** + * Rebuild sorted partition ranges after incremental updates. + * Should be called after add/drop partitions. + */ + public void rebuildSortedPartitionRanges() { + this.sortedPartitionRanges = buildSortedPartitionRanges(); + } + + /** + * Get sorted partition ranges for binary search filtering. + */ + public Optional> getSortedPartitionRanges() { + return Optional.ofNullable(sortedPartitionRanges); + } + + private SortedPartitionRanges buildSortedPartitionRanges() { + if (partitionNameToIdMap == null || partitionNameToIdMap.isEmpty()) { + return null; + } + + // Build name to partition item map for SortedPartitionRanges.buildFrom + BiMap idToName = partitionNameToIdMap.inverse(); + Map nameToPartitionItem = Maps.newHashMapWithExpectedSize(idToPartitionItem.size()); + for (Map.Entry entry : idToPartitionItem.entrySet()) { + String partitionName = idToName.get(entry.getKey()); + nameToPartitionItem.put(partitionName, entry.getValue()); + } + + return SortedPartitionRanges.build(nameToPartitionItem); + } } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SortedPartitionRanges.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SortedPartitionRanges.java index 50d4cb3befa538..0c4a1b034b5966 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SortedPartitionRanges.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SortedPartitionRanges.java @@ -20,9 +20,11 @@ import org.apache.doris.catalog.PartitionItem; import org.apache.doris.nereids.util.Utils; +import com.google.common.collect.Lists; import com.google.common.collect.Range; import java.util.List; +import java.util.Map; import java.util.Objects; /** SortedPartitionRanges */ @@ -41,6 +43,48 @@ public SortedPartitionRanges( ); } + /** + * Build SortedPartitionRanges from a partition map. + * This method extracts the common logic for building sorted partition ranges + * from both NereidsSortedPartitionsCacheManager and HiveMetaStoreCache. + * + * @param partitionMap map of partition id to partition item + * @return SortedPartitionRanges or null if the map is empty + */ + public static SortedPartitionRanges build(Map partitionMap) { + if (partitionMap == null || partitionMap.isEmpty()) { + return null; + } + + List> sortedRanges = Lists.newArrayListWithCapacity(partitionMap.size()); + List> defaultPartitions = Lists.newArrayList(); + + for (Map.Entry entry : partitionMap.entrySet()) { + PartitionItem partitionItem = entry.getValue(); + K id = entry.getKey(); + if (!partitionItem.isDefaultPartition()) { + List> ranges = PartitionItemToRange.toRanges(partitionItem); + for (Range range : ranges) { + sortedRanges.add(new PartitionItemAndRange<>(id, partitionItem, range)); + } + } else { + defaultPartitions.add(new PartitionItemAndId<>(id, partitionItem)); + } + } + + sortedRanges.sort((o1, o2) -> { + Range span1 = o1.range; + Range span2 = o2.range; + int result = span1.lowerEndpoint().compareTo(span2.lowerEndpoint()); + if (result != 0) { + return result; + } + return span1.upperEndpoint().compareTo(span2.upperEndpoint()); + }); + + return new SortedPartitionRanges<>(sortedRanges, defaultPartitions); + } + /** PartitionItemAndRange */ public static class PartitionItemAndRange { public final K id; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java index 0fb8e68c4863a8..7635ac803b5140 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java @@ -17,10 +17,7 @@ package org.apache.doris.nereids.rules.rewrite; -import org.apache.doris.catalog.Env; import org.apache.doris.catalog.PartitionItem; -import org.apache.doris.catalog.SupportBinarySearchFilteringPartitions; -import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; @@ -94,11 +91,10 @@ private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable, Map nameToPartitionItem = scan.getSelectedPartitions().selectedPartitions; Optional> sortedPartitionRanges = Optional.empty(); - if (externalTable instanceof SupportBinarySearchFilteringPartitions) { - NereidsSortedPartitionsCacheManager partitionsCacheManager = Env.getCurrentEnv() - .getSortedPartitionsCacheManager(); - sortedPartitionRanges = (Optional) partitionsCacheManager.get( - (SupportBinarySearchFilteringPartitions) externalTable, scan); + boolean enableBinarySearch = ctx.getConnectContext() == null + || ctx.getConnectContext().getSessionVariable().enableBinarySearchFilteringPartitions; + if (enableBinarySearch) { + sortedPartitionRanges = (Optional) externalTable.getSortedPartitionRanges(scan); } List prunedPartitions = new ArrayList<>(PartitionPruner.prune( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 2a7422cc55d093..2b6fa18828f200 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -34,7 +34,6 @@ import org.apache.doris.catalog.EsResource; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.PartitionItem; -import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ResourceMgr; import org.apache.doris.catalog.Type; @@ -54,21 +53,16 @@ import org.apache.doris.datasource.hive.HiveMetaStoreCache.HivePartitionValues; import org.apache.doris.datasource.hive.HiveMetaStoreCache.PartitionValueCacheKey; import org.apache.doris.mysql.privilege.Auth; -import org.apache.doris.planner.ColumnBound; import org.apache.doris.planner.ListPartitionPrunerV2; -import org.apache.doris.planner.PartitionPrunerV2Base.UniqueId; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSet; import org.apache.doris.utframe.TestWithFeService; import com.github.benmanes.caffeine.cache.LoadingCache; -import com.google.common.base.Preconditions; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Range; -import com.google.common.collect.RangeMap; import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -532,7 +526,6 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key, List // partition name format: nation=cn/city=beijing Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); - Map> idToUniqueIdsMap = Maps.newHashMapWithExpectedSize(partitionNames.size()); long idx = 0; for (String partitionName : partitionNames) { long partitionId = idx++; @@ -541,23 +534,8 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key, List partitionNameToIdMap.put(partitionName, partitionId); } - Map> uidToPartitionRange = null; - Map, UniqueId> rangeToId = null; - RangeMap singleColumnRangeMap = null; - Map> singleUidToColumnRangeMap = null; - if (key.getTypes().size() > 1) { - // uidToPartitionRange and rangeToId are only used for multi-column partition - uidToPartitionRange = ListPartitionPrunerV2.genUidToPartitionRange(idToPartitionItem, idToUniqueIdsMap); - rangeToId = ListPartitionPrunerV2.genRangeToId(uidToPartitionRange); - } else { - Preconditions.checkState(key.getTypes().size() == 1, key.getTypes()); - // singleColumnRangeMap is only used for single-column partition - singleColumnRangeMap = ListPartitionPrunerV2.genSingleColumnRangeMap(idToPartitionItem, idToUniqueIdsMap); - singleUidToColumnRangeMap = ListPartitionPrunerV2.genSingleUidToColumnRange(singleColumnRangeMap); - } Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - return new HivePartitionValues(idToPartitionItem, uidToPartitionRange, rangeToId, singleColumnRangeMap, - partitionNameToIdMap, idToUniqueIdsMap, singleUidToColumnRangeMap, partitionValuesMap); + return new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap); } @Test