Skip to content

Commit

Permalink
[fix](Nereids) column pruning under union broken unexpectedly (#26884)
Browse files Browse the repository at this point in the history 
introduced by PR #24060
  • Loading branch information
@morrySnow
morrySnow authored Nov 14, 2023
1 parent 9b3ddd1 commit 23e2bde
Show file tree
Hide file tree
Showing 27 changed files with 1,031 additions and 677 deletions.
3 changes: 0 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,6 @@ public Plan visitLogicalUnion(LogicalUnion union, PruneContext context) {
}

LogicalUnion prunedOutputUnion = pruneOutput(union, union.getOutputs(), union::pruneOutputs, context);
if (prunedOutputUnion == union) {
return union;
}

// start prune children of union
List<Slot> originOutput = union.getOutput();
Expand Down
63 changes: 39 additions & 24 deletions regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query14.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,24 +68,27 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
--------------------PhysicalProject
----------------------PhysicalOlapScan[store_sales]
--------------------PhysicalDistribute
----------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalProject
------------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
--------------------------PhysicalOlapScan[date_dim]
--------------PhysicalDistribute
----------------PhysicalProject
------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk))otherCondition=()
--------------------PhysicalProject
----------------------PhysicalOlapScan[catalog_sales]
--------------------PhysicalDistribute
----------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalProject
------------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
--------------------------PhysicalOlapScan[date_dim]
--------------PhysicalDistribute
----------------PhysicalProject
------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk))otherCondition=()
--------------------PhysicalProject
----------------------PhysicalOlapScan[web_sales]
--------------------PhysicalDistribute
----------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalProject
------------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
--------------------------PhysicalOlapScan[date_dim]
----PhysicalResultSink
------PhysicalTopN[MERGE_SORT]
--------PhysicalDistribute
Expand All @@ -107,14 +110,17 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
----------------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk))otherCondition=()
------------------------------------------PhysicalDistribute
--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk))otherCondition=()
----------------------------------------------PhysicalOlapScan[store_sales]
----------------------------------------------PhysicalProject
------------------------------------------------PhysicalOlapScan[store_sales]
----------------------------------------------PhysicalDistribute
------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001))
--------------------------------------------------PhysicalOlapScan[date_dim]
------------------------------------------------PhysicalProject
--------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001))
----------------------------------------------------PhysicalOlapScan[date_dim]
------------------------------------------PhysicalDistribute
--------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
----------------------------------------PhysicalDistribute
------------------------------------------PhysicalOlapScan[item]
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[item]
----------------------------PhysicalDistribute
------------------------------PhysicalAssertNumRows
--------------------------------PhysicalDistribute
Expand All @@ -130,14 +136,17 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
----------------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk))otherCondition=()
------------------------------------------PhysicalDistribute
--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk))otherCondition=()
----------------------------------------------PhysicalOlapScan[catalog_sales]
----------------------------------------------PhysicalProject
------------------------------------------------PhysicalOlapScan[catalog_sales]
----------------------------------------------PhysicalDistribute
------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001))
--------------------------------------------------PhysicalOlapScan[date_dim]
------------------------------------------------PhysicalProject
--------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001))
----------------------------------------------------PhysicalOlapScan[date_dim]
------------------------------------------PhysicalDistribute
--------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
----------------------------------------PhysicalDistribute
------------------------------------------PhysicalOlapScan[item]
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[item]
----------------------------PhysicalDistribute
------------------------------PhysicalAssertNumRows
--------------------------------PhysicalDistribute
Expand All @@ -153,14 +162,17 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
----------------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk))otherCondition=()
------------------------------------------PhysicalDistribute
--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk))otherCondition=()
----------------------------------------------PhysicalOlapScan[web_sales]
----------------------------------------------PhysicalProject
------------------------------------------------PhysicalOlapScan[web_sales]
----------------------------------------------PhysicalDistribute
------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001))
--------------------------------------------------PhysicalOlapScan[date_dim]
------------------------------------------------PhysicalProject
--------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001))
----------------------------------------------------PhysicalOlapScan[date_dim]
------------------------------------------PhysicalDistribute
--------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
----------------------------------------PhysicalDistribute
------------------------------------------PhysicalOlapScan[item]
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[item]
----------------------------PhysicalDistribute
------------------------------PhysicalAssertNumRows
--------------------------------PhysicalDistribute
Expand Down Expand Up @@ -235,24 +247,27 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
--------------------PhysicalProject
----------------------PhysicalOlapScan[store_sales]
--------------------PhysicalDistribute
----------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalProject
------------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
--------------------------PhysicalOlapScan[date_dim]
--------------PhysicalDistribute
----------------PhysicalProject
------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk))otherCondition=()
--------------------PhysicalProject
----------------------PhysicalOlapScan[catalog_sales]
--------------------PhysicalDistribute
----------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalProject
------------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
--------------------------PhysicalOlapScan[date_dim]
--------------PhysicalDistribute
----------------PhysicalProject
------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk))otherCondition=()
--------------------PhysicalProject
----------------------PhysicalOlapScan[web_sales]
--------------------PhysicalDistribute
----------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalProject
------------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999))
--------------------------PhysicalOlapScan[date_dim]
----PhysicalResultSink
------PhysicalTopN[MERGE_SORT]
--------PhysicalDistribute
Expand Down
68 changes: 40 additions & 28 deletions regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,32 +66,38 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------------------------PhysicalProject
--------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
----------------------PhysicalDistribute
------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk))otherCondition=()
--------------------------PhysicalDistribute
----------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk))otherCondition=()
------------------------------PhysicalOlapScan[catalog_sales]
------------------------------PhysicalDistribute
--------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
----------------------------------PhysicalOlapScan[date_dim]
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
------------------------PhysicalProject
--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk))otherCondition=()
----------------------------PhysicalDistribute
------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk))otherCondition=()
--------------------------------PhysicalProject
----------------------------------PhysicalOlapScan[catalog_sales]
--------------------------------PhysicalDistribute
----------------------------------PhysicalProject
------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
--------------------------------------PhysicalOlapScan[date_dim]
----------------------------PhysicalDistribute
------------------------------PhysicalProject
--------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
------------------PhysicalProject
--------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk))otherCondition=()
----------------------PhysicalDistribute
------------------------PhysicalProject
--------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
----------------------PhysicalDistribute
------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk))otherCondition=()
--------------------------PhysicalDistribute
----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk))otherCondition=()
------------------------------PhysicalOlapScan[web_sales]
------------------------------PhysicalDistribute
--------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
----------------------------------PhysicalOlapScan[date_dim]
--------------------------PhysicalDistribute
----------------------------PhysicalProject
------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
------------------------PhysicalProject
--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk))otherCondition=()
----------------------------PhysicalDistribute
------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk))otherCondition=()
--------------------------------PhysicalProject
----------------------------------PhysicalOlapScan[web_sales]
--------------------------------PhysicalDistribute
----------------------------------PhysicalProject
------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
--------------------------------------PhysicalOlapScan[date_dim]
----------------------------PhysicalDistribute
------------------------------PhysicalProject
--------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )

-- !ds_shape_23_2 --
PhysicalCteAnchor ( cteId=CTEId#0 )
Expand Down Expand Up @@ -165,15 +171,18 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
----------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk))otherCondition=()
------------------------------PhysicalDistribute
--------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk))otherCondition=()
----------------------------------PhysicalOlapScan[catalog_sales]
----------------------------------PhysicalProject
------------------------------------PhysicalOlapScan[catalog_sales]
----------------------------------PhysicalDistribute
------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
--------------------------------------PhysicalOlapScan[date_dim]
------------------------------------PhysicalProject
--------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
----------------------------------------PhysicalOlapScan[date_dim]
------------------------------PhysicalDistribute
--------------------------------PhysicalProject
----------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
----------------------------PhysicalDistribute
------------------------------PhysicalOlapScan[customer]
------------------------------PhysicalProject
--------------------------------PhysicalOlapScan[customer]
--------------hashAgg[GLOBAL]
----------------PhysicalDistribute
------------------hashAgg[LOCAL]
Expand All @@ -185,14 +194,17 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------------------------PhysicalDistribute
--------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk))otherCondition=()
----------------------------PhysicalDistribute
------------------------------PhysicalOlapScan[customer]
------------------------------PhysicalProject
--------------------------------PhysicalOlapScan[customer]
----------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk))otherCondition=()
------------------------------PhysicalDistribute
--------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk))otherCondition=()
----------------------------------PhysicalOlapScan[web_sales]
----------------------------------PhysicalProject
------------------------------------PhysicalOlapScan[web_sales]
----------------------------------PhysicalDistribute
------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
--------------------------------------PhysicalOlapScan[date_dim]
------------------------------------PhysicalProject
--------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
----------------------------------------PhysicalOlapScan[date_dim]
------------------------------PhysicalDistribute
--------------------------------PhysicalProject
----------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
Expand Down
Loading

0 comments on commit 23e2bde

Please sign in to comment.