From 3db7c7b6fdd6da3657e56cdd4bfc03ba6f25ff1b Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Wed, 29 Oct 2025 14:52:09 +0800 Subject: [PATCH 1/2] [opt](show-partitions) remove show partitions support for iceberg (#56985) Directly use `select * from table$partitions` to show the partition info of a iceberg table is enough. --- .../iceberg/IcebergExternalTable.java | 14 +-- .../maxcompute/MaxComputeExternalTable.java | 6 + .../paimon/PaimonExternalTable.java | 6 + .../data/mtmv_p0/test_iceberg_mtmv.out | 65 ++++++++--- .../suites/mtmv_p0/test_iceberg_mtmv.groovy | 105 ++++++++++-------- 5 files changed, 122 insertions(+), 74 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index 1a4ccfe87bbc48..d4032a8628ec4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -66,7 +66,6 @@ public class IcebergExternalTable extends ExternalTable implements MTMVRelatedTableIf, MTMVBaseTableIf, MvccTable { - private Table table; private boolean isValidRelatedTableCached = false; private boolean isValidRelatedTable = false; private boolean isView; @@ -89,11 +88,6 @@ protected synchronized void makeSureInitialized() { } } - @VisibleForTesting - public void setTable(Table table) { - this.table = table; - } - @Override public Optional initSchema(SchemaCacheKey key) { boolean isView = isView(); @@ -209,7 +203,7 @@ public boolean isValidRelatedTable() { } isValidRelatedTable = false; Set allFields = Sets.newHashSet(); - table = getIcebergTable(); + Table table = getIcebergTable(); for (PartitionSpec spec : table.specs().values()) { if (spec == null) { isValidRelatedTableCached = true; @@ -364,4 +358,10 @@ public Map properties() { } } + @Override + public boolean isPartitionedTable() { + makeSureInitialized(); + Table table = getIcebergTable(); + return table.spec().isPartitioned(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java index 7135ef96fdce9f..d9d167e786b7c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java @@ -345,4 +345,10 @@ public Table getOdpsTable() { return schemaCacheValue.map(value -> ((MaxComputeSchemaCacheValue) value).getOdpsTable()) .orElse(null); } + + @Override + public boolean isPartitionedTable() { + makeSureInitialized(); + return getOdpsTable().isPartitioned(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 06983786849428..add9413b49110a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -352,4 +352,10 @@ public Map getTableProperties() { return Collections.emptyMap(); } } + + @Override + public boolean isPartitionedTable() { + makeSureInitialized(); + return !paimonTable.partitionKeys().isEmpty(); + } } diff --git a/regression-test/data/mtmv_p0/test_iceberg_mtmv.out b/regression-test/data/mtmv_p0/test_iceberg_mtmv.out index 47d2d3a61a2eb0..666a587c99032d 100644 --- a/regression-test/data/mtmv_p0/test_iceberg_mtmv.out +++ b/regression-test/data/mtmv_p0/test_iceberg_mtmv.out @@ -59,10 +59,10 @@ \N 8 -- !test_iceberg_table_partition_ts -- -ts_day=20022 types: [DATETIMEV2]; keys: [2024-10-26 00:00:00]; types: [DATETIMEV2]; keys: [2024-10-27 00:00:00]; -ts_day=20023 types: [DATETIMEV2]; keys: [2024-10-27 00:00:00]; types: [DATETIMEV2]; keys: [2024-10-28 00:00:00]; -ts_day=20024 types: [DATETIMEV2]; keys: [2024-10-28 00:00:00]; types: [DATETIMEV2]; keys: [2024-10-29 00:00:00]; -ts_day=null types: [DATETIMEV2]; keys: [0000-01-01 00:00:00]; types: [DATETIMEV2]; keys: [0000-01-01 00:00:01]; +{"ts_day":null} +{"ts_day":"2024-10-26"} +{"ts_day":"2024-10-27"} +{"ts_day":"2024-10-28"} -- !unpartition -- 2024-10-26T01:02:03 1 @@ -110,9 +110,9 @@ ts_day=null types: [DATETIMEV2]; keys: [0000-01-01 00:00:00]; types: [DATETIMEV 2024-10-28 7 -- !test_iceberg_table_partition_d -- -d_month=655 types: [DATEV2]; keys: [2024-08-01]; types: [DATEV2]; keys: [2024-09-01]; -d_month=656 types: [DATEV2]; keys: [2024-09-01]; types: [DATEV2]; keys: [2024-10-01]; -d_month=657 types: [DATEV2]; keys: [2024-10-01]; types: [DATEV2]; keys: [2024-11-01]; +{"d_month":655} +{"d_month":656} +{"d_month":657} -- !refresh_one_partition -- 2024-01-01T00:00 4 @@ -130,21 +130,50 @@ d_month=657 types: [DATEV2]; keys: [2024-10-01]; types: [DATEV2]; keys: [2024-1 2024-01-02T00:00 3 -- !evolution2 -- -ts_day=20060 types: [DATETIMEV2]; keys: [2024-12-03 00:00:00]; types: [DATETIMEV2]; keys: [2024-12-04 00:00:00]; -ts_month=657 types: [DATETIMEV2]; keys: [2024-10-01 00:00:00]; types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; -ts_month=658 types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; types: [DATETIMEV2]; keys: [2024-12-01 00:00:00]; +{"ts_month":null, "ts_day":"2024-12-03"} +{"ts_month":657, "ts_day":null} +{"ts_month":658, "ts_day":null} -- !evolution3 -- -ts_day=20059 types: [DATETIMEV2]; keys: [2024-12-02 00:00:00]; types: [DATETIMEV2]; keys: [2024-12-03 00:00:00]; -ts_month=658 types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; types: [DATETIMEV2]; keys: [2024-12-01 00:00:00]; +{"ts_month":null, "ts_day":"2024-11-02"} +{"ts_month":null, "ts_day":"2024-11-03"} +{"ts_month":null, "ts_day":"2024-12-02"} +{"ts_month":658, "ts_day":null} -- !evolution4 -- -ts_month=657 types: [DATETIMEV2]; keys: [2024-10-01 00:00:00]; types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; -ts_month=658 types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; types: [DATETIMEV2]; keys: [2024-12-01 00:00:00]; +{"ts_month":null, "ts_day":"2024-11-02"} +{"ts_month":null, "ts_day":"2024-11-03"} +{"ts_month":657, "ts_day":null} +{"ts_month":658, "ts_day":null} -- !evolution5 -- -ts_day=20078 types: [DATETIMEV2]; keys: [2024-12-21 00:00:00]; types: [DATETIMEV2]; keys: [2024-12-22 00:00:00]; -ts_hour=482139 types: [DATETIMEV2]; keys: [2025-01-01 03:00:00]; types: [DATETIMEV2]; keys: [2025-01-01 04:00:00]; -ts_month=657 types: [DATETIMEV2]; keys: [2024-10-01 00:00:00]; types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; -ts_month=658 types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; types: [DATETIMEV2]; keys: [2024-12-01 00:00:00]; +{"ts_month":null, "ts_day":null, "ts_hour":481875} +{"ts_month":null, "ts_day":null, "ts_hour":482139} +{"ts_month":null, "ts_day":"2024-10-12", "ts_hour":null} +{"ts_month":null, "ts_day":"2024-12-21", "ts_hour":null} +{"ts_month":657, "ts_day":null, "ts_hour":null} +{"ts_month":658, "ts_day":null, "ts_hour":null} + +-- !test1 -- +{"create_date1_month":null, "create_date2_year":53} +{"create_date1_month":647, "create_date2_year":null} + +-- !test2 -- +{"create_date_bucket":2} +{"create_date_bucket":5} + +-- !test3 -- +{"create_date_month":647} + +-- !test4 -- +{"create_date1_month":647, "create_date2_year":null} +{"create_date1_month":647, "create_date2_year":53} + +-- !test5 -- +{"create_date_month":null} +{"create_date_month":647} + +-- !test6 -- +{"create_date1_month":647, "create_date2_year":null} +{"create_date1_month":647, "create_date2_year":53} diff --git a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy index f0019f2cb8b506..6bd694cab20a6c 100644 --- a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy @@ -137,7 +137,7 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh_null """select * from ${mvName1} order by value""" - qt_test_iceberg_table_partition_ts """show partitions from ${catalog_name}.${icebergDb}.${icebergTable1};""" + qt_test_iceberg_table_partition_ts """select `partition` from ${catalog_name}.${icebergDb}.${icebergTable1}\$partitions order by `partition`;""" def showPartitionsResult = sql """show partitions from ${mvName1}""" logger.info("showPartitionsResult: " + showPartitionsResult.toString()) @@ -198,7 +198,7 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh5 "select * from ${mvName2} order by value" - qt_test_iceberg_table_partition_d """show partitions from ${catalog_name}.${icebergDb}.${icebergTable2};""" + qt_test_iceberg_table_partition_d """select `partition` from ${catalog_name}.${icebergDb}.${icebergTable2}\$partitions order by `partition`;""" showPartitionsResult = sql """show partitions from ${mvName2}""" logger.info("showPartitionsResult: " + showPartitionsResult.toString()) @@ -241,7 +241,7 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ assertTrue(showPartitions.toString().contains("p_20240101000000_20240102000000")) assertTrue(showPartitions.toString().contains("p_20240102000000_20240103000000")) - // refresh one partiton + // refresh one partition sql """REFRESH MATERIALIZED VIEW ${mvName} partitions(p_20240101000000_20240102000000);""" waitingMTMVTaskFinishedByMvName(mvName, dbName) order_qt_refresh_one_partition "SELECT * FROM ${mvName} " @@ -265,52 +265,59 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ sql """drop table if exists ${catalog_name}.${icebergDb}.${icebergTable3}""" sql """use ${catalog_name}.test_db""" - qt_evolution2 "show partitions from replace_partition2" - qt_evolution3 "show partitions from replace_partition3" - qt_evolution4 "show partitions from replace_partition4" - qt_evolution5 "show partitions from replace_partition5" - - test { - sql "show partitions from replace_partition1" - // check exception message contains - exception "is not a supported partition table" - } - - test { - sql "show partitions from no_partition" - // check exception message contains - exception "is not a supported partition table" - } - - test { - sql "show partitions from not_support_trans" - // check exception message contains - exception "is not a supported partition table" - } - - test { - sql "show partitions from drop_partition1" - // check exception message contains - exception "is not a supported partition table" - } - - test { - sql "show partitions from drop_partition2" - // check exception message contains - exception "is not a supported partition table" - } - - test { - sql "show partitions from add_partition1" - // check exception message contains - exception "is not a supported partition table" - } - - test { - sql "show partitions from add_partition2" - // check exception message contains - exception "is not a supported partition table" - } + qt_evolution2 "select `partition` from replace_partition2\$partitions order by `partition`" + qt_evolution3 "select `partition` from replace_partition3\$partitions order by `partition`" + qt_evolution4 "select `partition` from replace_partition4\$partitions order by `partition`" + qt_evolution5 "select `partition` from replace_partition5\$partitions order by `partition`" + + qt_test1 "select `partition` from replace_partition1\$partitions order by `partition`"; + qt_test2 "select `partition` from not_support_trans\$partitions order by `partition`"; + qt_test3 "select `partition` from drop_partition1\$partitions order by `partition`"; + qt_test4 "select `partition` from drop_partition2\$partitions order by `partition`"; + qt_test5 "select `partition` from add_partition1\$partitions order by `partition`"; + qt_test6 "select `partition` from add_partition2\$partitions order by `partition`"; + + // test { + // sql "show partitions from replace_partition1" + // // check exception message contains + // exception "is not a supported partition table" + // } + + // test { + // sql "show partitions from no_partition" + // // check exception message contains + // exception "Unknown column 'partition'" + // } + + // test { + // sql "show partitions from not_support_trans" + // // check exception message contains + // exception "is not a supported partition table" + // } + + // test { + // sql "show partitions from drop_partition1" + // // check exception message contains + // exception "is not a supported partition table" + // } + + // test { + // sql "show partitions from drop_partition2" + // // check exception message contains + // exception "is not a supported partition table" + // } + + // test { + // sql "show partitions from add_partition1" + // // check exception message contains + // exception "is not a supported partition table" + // } + + // test { + // sql "show partitions from add_partition2" + // // check exception message contains + // exception "is not a supported partition table" + // } sql """ drop catalog if exists ${catalog_name} """ } From 1aae3d47fb2870a7c16a3ff93a39901cfb6b6884 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 31 Oct 2025 11:33:08 +0800 Subject: [PATCH 2/2] 2 --- .../data/mtmv_p0/test_iceberg_mtmv.out | 10 +++---- .../suites/mtmv_p0/test_iceberg_mtmv.groovy | 28 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/regression-test/data/mtmv_p0/test_iceberg_mtmv.out b/regression-test/data/mtmv_p0/test_iceberg_mtmv.out index 666a587c99032d..4a3a169a022c36 100644 --- a/regression-test/data/mtmv_p0/test_iceberg_mtmv.out +++ b/regression-test/data/mtmv_p0/test_iceberg_mtmv.out @@ -135,9 +135,9 @@ {"ts_month":658, "ts_day":null} -- !evolution3 -- +{"ts_month":null, "ts_day":"2024-12-02"} {"ts_month":null, "ts_day":"2024-11-02"} {"ts_month":null, "ts_day":"2024-11-03"} -{"ts_month":null, "ts_day":"2024-12-02"} {"ts_month":658, "ts_day":null} -- !evolution4 -- @@ -147,10 +147,10 @@ {"ts_month":658, "ts_day":null} -- !evolution5 -- -{"ts_month":null, "ts_day":null, "ts_hour":481875} -{"ts_month":null, "ts_day":null, "ts_hour":482139} {"ts_month":null, "ts_day":"2024-10-12", "ts_hour":null} +{"ts_month":null, "ts_day":null, "ts_hour":482139} {"ts_month":null, "ts_day":"2024-12-21", "ts_hour":null} +{"ts_month":null, "ts_day":null, "ts_hour":481875} {"ts_month":657, "ts_day":null, "ts_hour":null} {"ts_month":658, "ts_day":null, "ts_hour":null} @@ -166,14 +166,14 @@ {"create_date_month":647} -- !test4 -- -{"create_date1_month":647, "create_date2_year":null} {"create_date1_month":647, "create_date2_year":53} +{"create_date1_month":647, "create_date2_year":null} -- !test5 -- {"create_date_month":null} {"create_date_month":647} -- !test6 -- -{"create_date1_month":647, "create_date2_year":null} {"create_date1_month":647, "create_date2_year":53} +{"create_date1_month":647, "create_date2_year":null} diff --git a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy index 6bd694cab20a6c..edbd9a34633c4d 100644 --- a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy @@ -137,7 +137,7 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh_null """select * from ${mvName1} order by value""" - qt_test_iceberg_table_partition_ts """select `partition` from ${catalog_name}.${icebergDb}.${icebergTable1}\$partitions order by `partition`;""" + qt_test_iceberg_table_partition_ts """select `partition` from ${catalog_name}.${icebergDb}.${icebergTable1}\$partitions order by struct_element(`partition`, 1);""" def showPartitionsResult = sql """show partitions from ${mvName1}""" logger.info("showPartitionsResult: " + showPartitionsResult.toString()) @@ -198,7 +198,7 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh5 "select * from ${mvName2} order by value" - qt_test_iceberg_table_partition_d """select `partition` from ${catalog_name}.${icebergDb}.${icebergTable2}\$partitions order by `partition`;""" + qt_test_iceberg_table_partition_d """select `partition` from ${catalog_name}.${icebergDb}.${icebergTable2}\$partitions order by struct_element(`partition`, 1);""" showPartitionsResult = sql """show partitions from ${mvName2}""" logger.info("showPartitionsResult: " + showPartitionsResult.toString()) @@ -265,17 +265,17 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ sql """drop table if exists ${catalog_name}.${icebergDb}.${icebergTable3}""" sql """use ${catalog_name}.test_db""" - qt_evolution2 "select `partition` from replace_partition2\$partitions order by `partition`" - qt_evolution3 "select `partition` from replace_partition3\$partitions order by `partition`" - qt_evolution4 "select `partition` from replace_partition4\$partitions order by `partition`" - qt_evolution5 "select `partition` from replace_partition5\$partitions order by `partition`" - - qt_test1 "select `partition` from replace_partition1\$partitions order by `partition`"; - qt_test2 "select `partition` from not_support_trans\$partitions order by `partition`"; - qt_test3 "select `partition` from drop_partition1\$partitions order by `partition`"; - qt_test4 "select `partition` from drop_partition2\$partitions order by `partition`"; - qt_test5 "select `partition` from add_partition1\$partitions order by `partition`"; - qt_test6 "select `partition` from add_partition2\$partitions order by `partition`"; + qt_evolution2 "select `partition` from replace_partition2\$partitions order by struct_element(`partition`, 1)" + qt_evolution3 "select `partition` from replace_partition3\$partitions order by struct_element(`partition`, 1)" + qt_evolution4 "select `partition` from replace_partition4\$partitions order by struct_element(`partition`, 1)" + qt_evolution5 "select `partition` from replace_partition5\$partitions order by struct_element(`partition`, 1)" + + qt_test1 "select `partition` from replace_partition1\$partitions order by struct_element(`partition`, 1)"; + qt_test2 "select `partition` from not_support_trans\$partitions order by struct_element(`partition`, 1)"; + qt_test3 "select `partition` from drop_partition1\$partitions order by struct_element(`partition`, 1)"; + qt_test4 "select `partition` from drop_partition2\$partitions order by struct_element(`partition`, 1)"; + qt_test5 "select `partition` from add_partition1\$partitions order by struct_element(`partition`, 1)"; + qt_test6 "select `partition` from add_partition2\$partitions order by struct_element(`partition`, 1)"; // test { // sql "show partitions from replace_partition1" @@ -319,7 +319,7 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ // exception "is not a supported partition table" // } - sql """ drop catalog if exists ${catalog_name} """ + // sql """ drop catalog if exists ${catalog_name} """ } }