diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java index 1249dd0577ca69..04785e1db14937 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java @@ -60,4 +60,9 @@ public List> getOnePartitionInputs() { public Expression evaluate(Expression expression, Map currentInputs) { return BooleanLiteral.TRUE; } + + @Override + public boolean isDefaultPartition() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java index fcdd9c2a453f89..dd71ed8e99ff76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java @@ -98,4 +98,9 @@ public Expression visitSlot(Slot slot, Map context) { public Expression evaluate(Expression expression, Map currentInputs) { return expression.accept(this, currentInputs); } + + @Override + public boolean isDefaultPartition() { + return partitionItem.isDefaultPartition(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OnePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OnePartitionEvaluator.java index a2e5c332a75964..c51252b44a624d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OnePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OnePartitionEvaluator.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import java.util.List; import java.util.Map; @@ -45,4 +46,13 @@ public interface OnePartitionEvaluator { * we will return a context which result expression is BooleanLiteral.FALSE */ Expression evaluate(Expression expression, Map currentInputs); + + default Expression evaluateWithDefaultPartition(Expression expression, Map inputs) { + if (isDefaultPartition()) { + return BooleanLiteral.TRUE; + } + return evaluate(expression, inputs); + } + + boolean isDefaultPartition(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index d03d3276eff4bf..f98576468523eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -665,4 +665,9 @@ public EvaluateRangeResult(Expression result, Map columnRange this.childrenResult = childrenResult; } } + + @Override + public boolean isDefaultPartition() { + return partitionItem.isDefaultPartition(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java index 7e89a97915ae7b..8479c27776ab2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java @@ -83,6 +83,7 @@ public static List prune(List partitionSlots, Expression partitionPr .collect(ImmutableList.toImmutableList()); PartitionPruner partitionPruner = new PartitionPruner(evaluators, partitionPredicate); + //TODO: we keep default partition because it's too hard to prune it, we return false in canPrune(). return partitionPruner.prune(); } @@ -110,7 +111,7 @@ public static final OnePartitionEvaluator toPartitionEvaluator(long id, Partitio private boolean canPrune(OnePartitionEvaluator evaluator) { List> onePartitionInputs = evaluator.getOnePartitionInputs(); for (Map currentInputs : onePartitionInputs) { - Expression result = evaluator.evaluate(partitionPredicate, currentInputs); + Expression result = evaluator.evaluateWithDefaultPartition(partitionPredicate, currentInputs); if (!result.equals(BooleanLiteral.FALSE)) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/UnknownPartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/UnknownPartitionEvaluator.java index 1325635eb740c9..ae313ca09de269 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/UnknownPartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/UnknownPartitionEvaluator.java @@ -52,4 +52,9 @@ public Expression evaluate(Expression expression, Map // do not prune return expression; } + + @Override + public boolean isDefaultPartition() { + return partitionItem.isDefaultPartition(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java index 4c10f339bf9ef6..1bdff7f6c10eba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java @@ -68,6 +68,7 @@ public Rule build() { List prunedPartitions = new ArrayList<>(PartitionPruner.prune( partitionSlots, filter.getPredicate(), partitionInfo, ctx.cascadesContext, PartitionTableType.OLAP)); + List manuallySpecifiedPartitions = scan.getManuallySpecifiedPartitions(); if (!CollectionUtils.isEmpty(manuallySpecifiedPartitions)) { prunedPartitions.retainAll(manuallySpecifiedPartitions); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartitionTest.java index 829feda16cb19f..85fde57b517fa2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartitionTest.java @@ -196,6 +196,28 @@ void testOlapScanPartitionPruneWithMultiColumnCase() throws Exception { test("testOlapScanPartitionPruneWithMultiColumnCase", "cast(col1 as bigint) + 1 = 5", 1); } + @Test + public void prunePartitionWithDefaultPartition() throws Exception { + createTable("CREATE TABLE IF NOT EXISTS test_default_in_parts (\n" + + " k1 tinyint NOT NULL, \n" + + " k2 smallint NOT NULL, \n" + + " k3 int NOT NULL, \n" + + " k4 bigint NOT NULL, \n" + + " k5 decimal(9, 3) NOT NULL,\n" + + " k8 double max NOT NULL, \n" + + " k9 float sum NOT NULL ) \n" + + " AGGREGATE KEY(k1,k2,k3,k4,k5)\n" + + " PARTITION BY LIST(k1) ( \n" + + " PARTITION p1 VALUES IN (\"1\",\"2\",\"3\",\"4\"), \n" + + " PARTITION p2 VALUES IN (\"5\",\"6\",\"7\",\"8\"), \n" + + " PARTITION p3 ) \n" + + " DISTRIBUTED BY HASH(k1) BUCKETS 5 properties(\"replication_num\" = \"1\")"); + test("test_default_in_parts", "k1 = 10", 1); + test("test_default_in_parts", "k1 > 5", 2); + test("test_default_in_parts", "k1 > 2", 3); + test("test_default_in_parts", "(k1 > 1 and k1 < 8)", 3); + } + @Test public void prunePartitionWithOrPredicate() { test("test_list_parts", "(part = 9 and id <= 500) or (part = 3)", 1);