From 7fd2cf6fa8ae2121aa97cb59a5abd124fd7206b5 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 10 Dec 2025 12:04:26 +0800 Subject: [PATCH 1/3] [opt](iceberg) able to ignore iceberg dangling delete to do count push down --- .../iceberg/source/IcebergScanNode.java | 21 +++++++++++++------ .../org/apache/doris/qe/SessionVariable.java | 13 ++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index f5208397a0f324..9dc70289bf66d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -584,15 +584,24 @@ public long getCountFromSnapshot() throws UserException { return 0; } - // `TOTAL_POSITION_DELETES` is need to 0, - // because prevent 'dangling delete' problem after `rewrite_data_files` - // ref: https://iceberg.apache.org/docs/nightly/spark-procedures/#rewrite_position_delete_files Map summary = snapshot.summary(); - if (!summary.get(IcebergUtils.TOTAL_EQUALITY_DELETES).equals("0") - || !summary.get(IcebergUtils.TOTAL_POSITION_DELETES).equals("0")) { + if (!summary.get(IcebergUtils.TOTAL_EQUALITY_DELETES).equals("0")) { + // has equality delete files, can not push down count + return -1; + } + + long deleteCount = Long.parseLong(summary.get(IcebergUtils.TOTAL_POSITION_DELETES)); + if (deleteCount == 0) { + // no delete files, can push down count directly + return Long.parseLong(summary.get(IcebergUtils.TOTAL_RECORDS)); + } + if (sessionVariable.ignore_iceberg_dangling_delete) { + // has position delete files, if we ignore dangling delete, can push down count + return Long.parseLong(summary.get(IcebergUtils.TOTAL_RECORDS)) - deleteCount; + } else { + // otherwise, can not push down count return -1; } - return Long.parseLong(summary.get(IcebergUtils.TOTAL_RECORDS)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 5e51b0ee374277..85b4268d764aeb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -3181,6 +3181,19 @@ public boolean isEnableESParallelScroll() { ) public boolean useV3StorageFormat = false; + public static final String IGNORE_ICEBERG_DANGLING_DELETE = "ignore_iceberg_dangling_delete"; + @VariableMgr.VarAttr(name = IGNORE_ICEBERG_DANGLING_DELETE, + description = {"是否忽略 Iceberg 表中 dangling delete 文件对 COUNT(*) 统计信息的影响。" + + "默认为 true，COUNT(*) 会直接从元信息中获取行数，性能更好，但是如果有 dangling delete，结果可能是不准确的。" + + "设置为 false 时，COUNT(*) 会扫描数据文件以排除 dangling delete 文件的影响。", + " Whether to ignore the impact of dangling delete files in Iceberg tables on COUNT(*) statistics. " + + "The default is true, COUNT(*) will directly obtain the number of rows from metadata, " + + "which has better performance, but if there are dangling deletes, " + + "the result may be inaccurate. " + + "When set to false, COUNT(*) will scan data files " + + "to exclude the impact of dangling delete files."}) + public boolean ignore_iceberg_dangling_delete = false; + // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. @SuppressWarnings("checkstyle:Indentation") From aa7c17be6afda5d780918270c14a90e85b60240b Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 10 Dec 2025 12:29:37 +0800 Subject: [PATCH 2/3] 2 --- .../iceberg/test_iceberg_optimize_count.out | 6 ++++++ .../iceberg/test_iceberg_optimize_count.groovy | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_optimize_count.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_optimize_count.out index 20d03ad9c06bec..a64d8480129d8b 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_optimize_count.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_optimize_count.out @@ -38,3 +38,9 @@ -- !q09 -- 2 +-- !sql_count1 -- +2 + +-- !sql_count1 -- +1 + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy index 0f1f5535c058e9..4815de33584b5a 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy @@ -162,10 +162,19 @@ suite("test_iceberg_optimize_count", "p0,external,doris,external_docker,external qt_q09 """${sqlstr5}""" + sql """ set ignore_iceberg_dangling_delete=false""" explain { sql("""${sqlstr5}""") contains """pushdown agg=COUNT (-1)""" } + qt_sql_count1 """select count(*) from ${catalog_name}.test_db.dangling_delete_after_write;""" + + sql """ set ignore_iceberg_dangling_delete=true""" + explain { + sql("""${sqlstr5}""") + contains """pushdown agg=COUNT (1)""" + } + qt_sql_count1 """select count(*) from ${catalog_name}.test_db.dangling_delete_after_write;""" } finally { sql """ set enable_count_push_down_for_external_table=true; """ From a6bb8511e5260b043c23444ca19867b87f91951e Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 10 Dec 2025 12:34:58 +0800 Subject: [PATCH 3/3] 3 --- .../apache/doris/datasource/iceberg/source/IcebergScanNode.java | 2 +- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 9dc70289bf66d0..61b1bf9c8ab1b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -595,7 +595,7 @@ public long getCountFromSnapshot() throws UserException { // no delete files, can push down count directly return Long.parseLong(summary.get(IcebergUtils.TOTAL_RECORDS)); } - if (sessionVariable.ignore_iceberg_dangling_delete) { + if (sessionVariable.ignoreIcebergDanglingDelete) { // has position delete files, if we ignore dangling delete, can push down count return Long.parseLong(summary.get(IcebergUtils.TOTAL_RECORDS)) - deleteCount; } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 85b4268d764aeb..6b59f34cb504a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -3192,7 +3192,7 @@ public boolean isEnableESParallelScroll() { + "the result may be inaccurate. " + "When set to false, COUNT(*) will scan data files " + "to exclude the impact of dangling delete files."}) - public boolean ignore_iceberg_dangling_delete = false; + public boolean ignoreIcebergDanglingDelete = false; // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code.