diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql index 7aa4170eab0985..7722d09463686e 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql @@ -22,4 +22,49 @@ insert into test_tb_mix_format values (1,1,'b'),(2,1,'b'),(3,1,'b'),(4,1,'b'),(5 -- update some data, these splits will be readed by jni insert into test_tb_mix_format values (1,2,'b'),(2,2,'b'),(3,2,'b'),(4,2,'b'),(5,2,'b'); -- delete foramt in table properties, doris should get format by file name -alter table test_tb_mix_format unset TBLPROPERTIES ('file.format'); \ No newline at end of file +alter table test_tb_mix_format unset TBLPROPERTIES ('file.format'); + +drop table if exists two_partition; +CREATE TABLE two_partition ( + id BIGINT, + create_date STRING, + region STRING +) PARTITIONED BY (create_date,region) TBLPROPERTIES ( + 'primary-key' = 'create_date,region,id', + 'bucket'=10, + 'file.format'='orc' +); + +insert into two_partition values(1,'2020-01-01','bj'); +insert into two_partition values(2,'2020-01-01','sh'); +insert into two_partition values(3,'2038-01-01','bj'); +insert into two_partition values(4,'2038-01-01','sh'); +insert into two_partition values(5,'2038-01-02','bj'); + +drop table if exists null_partition; +CREATE TABLE null_partition ( + id BIGINT, + region STRING +) PARTITIONED BY (region) TBLPROPERTIES ( + 'primary-key' = 'region,id', + 'bucket'=10, + 'file.format'='orc' +); +-- null NULL "null" all will be in partition [null] +insert into null_partition values(1,'bj'); +insert into null_partition values(2,null); +insert into null_partition values(3,NULL); +insert into null_partition values(4,'null'); +insert into null_partition values(5,'NULL'); + +drop table if exists date_partition; +CREATE TABLE date_partition ( + id BIGINT, + create_date DATE +) PARTITIONED BY (create_date) TBLPROPERTIES ( + 'primary-key' = 'create_date,id', + 'bucket'=10, + 'file.format'='orc' +); + +insert into date_partition values(1,date '2020-01-01'); \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 3f22ce4c46b32f..efd8e6bb7a7a42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -144,9 +144,17 @@ public long fetchRowCount() { @Override public List getPartitionColumns(Optional snapshot) { + if (isPartitionInvalid(snapshot)) { + return Collections.emptyList(); + } return getPaimonSchemaCacheValue(snapshot).getPartitionColumns(); } + private boolean isPartitionInvalid(Optional snapshot) { + PaimonSnapshotCacheValue paimonSnapshotCacheValue = getOrFetchSnapshotCacheValue(snapshot); + return paimonSnapshotCacheValue.getPartitionInfo().isPartitionInvalid(); + } + @Override public MvccSnapshot loadSnapshot() { return new PaimonMvccSnapshot(getPaimonSnapshotCacheValue()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 4d3326f8e48376..88515a2510d2c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -45,4 +45,9 @@ public Map getNameToPartitionItem() { public Map getNameToPartition() { return nameToPartition; } + + public boolean isPartitionInvalid() { + // when transfer to partitionItem failed, will not equal + return nameToPartitionItem.size() != nameToPartition.size(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 1f7576dca51d93..b3df41bc5cef17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -117,7 +117,7 @@ public static PaimonPartition rowToPartition(InternalRow row) { } public static PaimonPartitionInfo generatePartitionInfo(List partitionColumns, - List paimonPartitions) throws AnalysisException { + List paimonPartitions) { Map nameToPartitionItem = Maps.newHashMap(); Map nameToPartition = Maps.newHashMap(); PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(nameToPartitionItem, nameToPartition); @@ -127,7 +127,14 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo for (PaimonPartition paimonPartition : paimonPartitions) { String partitionName = getPartitionName(partitionColumns, paimonPartition.getPartitionValues()); nameToPartition.put(partitionName, paimonPartition); - nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + try { + // partition values return by paimon api, may have problem, + // to avoid affecting the query, we catch exceptions here + nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + } catch (Exception e) { + LOG.warn("toListPartitionItem failed, partitionColumns: {}, partitionValues: {}", partitionColumns, + paimonPartition.getPartitionValues(), e); + } } return partitionInfo; }