Skip to content

Commit

Permalink
[fix](Nereids) default partition be prunned by mistake (#24186)
Browse files Browse the repository at this point in the history
```sql
CREATE TABLE IF NOT EXISTS t ( 
            k1 tinyint NOT NULL, 
            k2 smallint NOT NULL, 
            k3 int NOT NULL, 
            k4 bigint NOT NULL, 
            k5 decimal(9, 3) NOT NULL,
            k8 double max NOT NULL, 
            k9 float sum NOT NULL ) 
        AGGREGATE KEY(k1,k2,k3,k4,k5)
        PARTITION BY LIST(k1) ( 
            PARTITION p1 VALUES IN ("1","2","3","4"), 
            PARTITION p2 VALUES IN ("5","6","7","8"), 
            PARTITION p3 ) 
        DISTRIBUTED BY HASH(k1) BUCKETS 5 properties("replication_num" = "1")

select * from t where k1=10
```
The query will return 0 rows because p3 is pruned, we fix it by skip prune default partitions.

TODO: prune default partition if filter do not hit it
  • Loading branch information
sohardforaname authored Sep 13, 2023
1 parent a6f05e8 commit f985b28
Show file tree
Hide file tree
Showing 8 changed files with 55 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,9 @@ public List<Map<Slot, PartitionSlotInput>> getOnePartitionInputs() {
public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) {
return BooleanLiteral.TRUE;
}

@Override
public boolean isDefaultPartition() {
return true;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -98,4 +98,9 @@ public Expression visitSlot(Slot slot, Map<Slot, PartitionSlotInput> context) {
public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) {
return expression.accept(this, currentInputs);
}

@Override
public boolean isDefaultPartition() {
return partitionItem.isDefaultPartition();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;

import java.util.List;
import java.util.Map;
Expand All @@ -45,4 +46,13 @@ public interface OnePartitionEvaluator {
* we will return a context which result expression is BooleanLiteral.FALSE
*/
Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs);

default Expression evaluateWithDefaultPartition(Expression expression, Map<Slot, PartitionSlotInput> inputs) {
if (isDefaultPartition()) {
return BooleanLiteral.TRUE;
}
return evaluate(expression, inputs);
}

boolean isDefaultPartition();
}
Original file line number Diff line number Diff line change
Expand Up @@ -665,4 +665,9 @@ public EvaluateRangeResult(Expression result, Map<Slot, ColumnRange> columnRange
this.childrenResult = childrenResult;
}
}

@Override
public boolean isDefaultPartition() {
return partitionItem.isDefaultPartition();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public static List<Long> prune(List<Slot> partitionSlots, Expression partitionPr
.collect(ImmutableList.toImmutableList());

PartitionPruner partitionPruner = new PartitionPruner(evaluators, partitionPredicate);
//TODO: we keep default partition because it's too hard to prune it, we return false in canPrune().
return partitionPruner.prune();
}

Expand Down Expand Up @@ -110,7 +111,7 @@ public static final OnePartitionEvaluator toPartitionEvaluator(long id, Partitio
private boolean canPrune(OnePartitionEvaluator evaluator) {
List<Map<Slot, PartitionSlotInput>> onePartitionInputs = evaluator.getOnePartitionInputs();
for (Map<Slot, PartitionSlotInput> currentInputs : onePartitionInputs) {
Expression result = evaluator.evaluate(partitionPredicate, currentInputs);
Expression result = evaluator.evaluateWithDefaultPartition(partitionPredicate, currentInputs);
if (!result.equals(BooleanLiteral.FALSE)) {
return false;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,9 @@ public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput>
// do not prune
return expression;
}

@Override
public boolean isDefaultPartition() {
return partitionItem.isDefaultPartition();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ public Rule build() {
List<Long> prunedPartitions = new ArrayList<>(PartitionPruner.prune(
partitionSlots, filter.getPredicate(), partitionInfo, ctx.cascadesContext,
PartitionTableType.OLAP));

List<Long> manuallySpecifiedPartitions = scan.getManuallySpecifiedPartitions();
if (!CollectionUtils.isEmpty(manuallySpecifiedPartitions)) {
prunedPartitions.retainAll(manuallySpecifiedPartitions);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,28 @@ void testOlapScanPartitionPruneWithMultiColumnCase() throws Exception {
test("testOlapScanPartitionPruneWithMultiColumnCase", "cast(col1 as bigint) + 1 = 5", 1);
}

@Test
public void prunePartitionWithDefaultPartition() throws Exception {
createTable("CREATE TABLE IF NOT EXISTS test_default_in_parts (\n"
+ " k1 tinyint NOT NULL, \n"
+ " k2 smallint NOT NULL, \n"
+ " k3 int NOT NULL, \n"
+ " k4 bigint NOT NULL, \n"
+ " k5 decimal(9, 3) NOT NULL,\n"
+ " k8 double max NOT NULL, \n"
+ " k9 float sum NOT NULL ) \n"
+ " AGGREGATE KEY(k1,k2,k3,k4,k5)\n"
+ " PARTITION BY LIST(k1) ( \n"
+ " PARTITION p1 VALUES IN (\"1\",\"2\",\"3\",\"4\"), \n"
+ " PARTITION p2 VALUES IN (\"5\",\"6\",\"7\",\"8\"), \n"
+ " PARTITION p3 ) \n"
+ " DISTRIBUTED BY HASH(k1) BUCKETS 5 properties(\"replication_num\" = \"1\")");
test("test_default_in_parts", "k1 = 10", 1);
test("test_default_in_parts", "k1 > 5", 2);
test("test_default_in_parts", "k1 > 2", 3);
test("test_default_in_parts", "(k1 > 1 and k1 < 8)", 3);
}

@Test
public void prunePartitionWithOrPredicate() {
test("test_list_parts", "(part = 9 and id <= 500) or (part = 3)", 1);
Expand Down

0 comments on commit f985b28

Please sign in to comment.