diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 312f2382b0d0f5..ad36dc221d8b3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -34,6 +34,7 @@ import org.apache.doris.common.security.authentication.AuthenticationConfig; import org.apache.doris.common.util.CacheBulkLoader; import org.apache.doris.common.util.LocationPath; +import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo; import org.apache.doris.datasource.property.PropertyConverter; @@ -250,9 +251,8 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); Map> idToUniqueIdsMap = Maps.newHashMapWithExpectedSize(partitionNames.size()); - long idx = 0; for (String partitionName : partitionNames) { - long partitionId = idx++; + long partitionId = Util.genIdByName(catalog.getName(), key.dbName, key.tblName, partitionName); ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types); idToPartitionItem.put(partitionId, listPartitionItem); partitionNameToIdMap.put(partitionName, partitionId); @@ -273,7 +273,7 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { singleUidToColumnRangeMap = ListPartitionPrunerV2.genSingleUidToColumnRange(singleColumnRangeMap); } Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - return new HivePartitionValues(idToPartitionItem, uidToPartitionRange, rangeToId, singleColumnRangeMap, idx, + return new HivePartitionValues(idToPartitionItem, uidToPartitionRange, rangeToId, singleColumnRangeMap, partitionNameToIdMap, idToUniqueIdsMap, singleUidToColumnRangeMap, partitionValuesMap); } @@ -638,13 +638,12 @@ public void addPartitionsCache(String dbName, String tblName, List parti Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); Map> idToUniqueIdsMap = copy.getIdToUniqueIdsMap(); Map idToPartitionItem = new HashMap<>(); - long idx = copy.getNextPartitionId(); for (String partitionName : partitionNames) { if (partitionNameToIdMapBefore.containsKey(partitionName)) { LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", partitionName, tblName); continue; } - long partitionId = idx++; + long partitionId = Util.genIdByName(catalog.getName(), dbName, tblName, partitionName); ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types); idToPartitionItemBefore.put(partitionId, listPartitionItem); idToPartitionItem.put(partitionId, listPartitionItem); @@ -653,7 +652,6 @@ public void addPartitionsCache(String dbName, String tblName, List parti Map> partitionValuesMapBefore = copy.getPartitionValuesMap(); Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); partitionValuesMapBefore.putAll(partitionValuesMap); - copy.setNextPartitionId(idx); if (key.types.size() > 1) { Map> uidToPartitionRangeBefore = copy.getUidToPartitionRange(); // uidToPartitionRange and rangeToId are only used for multi-column partition @@ -1075,7 +1073,6 @@ public static class HiveFileStatus { @Data public static class HivePartitionValues { - private long nextPartitionId; private BiMap partitionNameToIdMap; private Map> idToUniqueIdsMap; private Map idToPartitionItem; @@ -1094,7 +1091,6 @@ public HivePartitionValues(Map idToPartitionItem, Map> uidToPartitionRange, Map, UniqueId> rangeToId, RangeMap singleColumnRangeMap, - long nextPartitionId, BiMap partitionNameToIdMap, Map> idToUniqueIdsMap, Map> singleUidToColumnRangeMap, @@ -1103,7 +1099,6 @@ public HivePartitionValues(Map idToPartitionItem, this.uidToPartitionRange = uidToPartitionRange; this.rangeToId = rangeToId; this.singleColumnRangeMap = singleColumnRangeMap; - this.nextPartitionId = nextPartitionId; this.partitionNameToIdMap = partitionNameToIdMap; this.idToUniqueIdsMap = idToUniqueIdsMap; this.singleUidToColumnRangeMap = singleUidToColumnRangeMap; @@ -1112,7 +1107,6 @@ public HivePartitionValues(Map idToPartitionItem, public HivePartitionValues copy() { HivePartitionValues copy = new HivePartitionValues(); - copy.setNextPartitionId(nextPartitionId); copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap)); copy.setIdToUniqueIdsMap(idToUniqueIdsMap == null ? null : Maps.newHashMap(idToUniqueIdsMap)); copy.setIdToPartitionItem(idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 5f1e19f32846f2..e5e8a9d71a5baf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -548,7 +548,7 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key, List singleUidToColumnRangeMap = ListPartitionPrunerV2.genSingleUidToColumnRange(singleColumnRangeMap); } Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - return new HivePartitionValues(idToPartitionItem, uidToPartitionRange, rangeToId, singleColumnRangeMap, idx, + return new HivePartitionValues(idToPartitionItem, uidToPartitionRange, rangeToId, singleColumnRangeMap, partitionNameToIdMap, idToUniqueIdsMap, singleUidToColumnRangeMap, partitionValuesMap); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java index 6c1accce675fe8..5af7b831e98c86 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java @@ -103,8 +103,8 @@ public List listPartitionNames(String dbName, String tblName) { String tblName = "tb"; HiveMetaStoreCache.HivePartitionValues partitionValues = cache.getPartitionValues(dbName, tblName, types); Assert.assertEquals(1, partitionValues.getIdToPartitionItem().size()); - Assert.assertTrue(partitionValues.getIdToPartitionItem().containsKey(0L)); - List items = partitionValues.getIdToPartitionItem().get(0L).getItems(); + Assert.assertTrue(partitionValues.getIdToPartitionItem().containsKey(8882801933302843777L)); + List items = partitionValues.getIdToPartitionItem().get(8882801933302843777L).getItems(); Assert.assertEquals(1, items.size()); PartitionKey partitionKey = items.get(0); Assert.assertEquals("1.234", partitionKey.getKeys().get(0).toString()); @@ -116,8 +116,8 @@ public List listPartitionNames(String dbName, String tblName) { cache.addPartitionsCache(dbName, tblName, values, types); HiveMetaStoreCache.HivePartitionValues partitionValues2 = cache.getPartitionValues(dbName, tblName, types); Assert.assertEquals(2, partitionValues2.getIdToPartitionItem().size()); - Assert.assertTrue(partitionValues2.getIdToPartitionItem().containsKey(1L)); - List items2 = partitionValues2.getIdToPartitionItem().get(1L).getItems(); + Assert.assertTrue(partitionValues2.getIdToPartitionItem().containsKey(7070400225537799947L)); + List items2 = partitionValues2.getIdToPartitionItem().get(7070400225537799947L).getItems(); Assert.assertEquals(1, items2.size()); PartitionKey partitionKey2 = items2.get(0); Assert.assertEquals("5.678", partitionKey2.getKeys().get(0).toString()); @@ -128,8 +128,8 @@ public List listPartitionNames(String dbName, String tblName) { cache.invalidateTableCache(dbName, tblName); HiveMetaStoreCache.HivePartitionValues partitionValues3 = cache.getPartitionValues(dbName, tblName, types); Assert.assertEquals(1, partitionValues3.getIdToPartitionItem().size()); - Assert.assertTrue(partitionValues3.getIdToPartitionItem().containsKey(0L)); - List items3 = partitionValues3.getIdToPartitionItem().get(0L).getItems(); + Assert.assertTrue(partitionValues3.getIdToPartitionItem().containsKey(8882801933302843777L)); + List items3 = partitionValues3.getIdToPartitionItem().get(8882801933302843777L).getItems(); Assert.assertEquals(1, items3.size()); PartitionKey partitionKey3 = items3.get(0); Assert.assertEquals("1.234", partitionKey3.getKeys().get(0).toString());