Skip to content

Commit e7e33ec

Browse files
frankmcsherryfrankmcsherry
committed
Rebase onto Analysis prototype
1 parent 7642475 commit e7e33ec

15 files changed

+1020
-584
lines changed

‎src/transform/src/analysis.rs

+2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99

1010
//! Traits and types for reusable expression analysis
1111
12+
pub mod equivalences;
13+
1214
use mz_expr::MirRelationExpr;
1315

1416
pub use common::{Derived, DerivedBuilder, DerivedView};

‎src/transform/src/analysis/equivalences.rs

+560
Large diffs are not rendered by default.

‎src/transform/src/attribute/analysis.rs

Whitespace-only changes.

‎src/transform/src/attribute/mod.rs

+1-3
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,7 @@ pub trait Attribute {
5353
fn handle_env_tasks(&mut self) {}
5454

5555
/// The attribute ids of all other attributes that this attribute depends on.
56-
fn add_dependencies(builder: &mut DerivedAttributesBuilder)
57-
where
58-
Self: Sized;
56+
fn add_dependencies(builder: &mut DerivedAttributesBuilder);
5957

6058
/// Attributes for each subexpression, visited in post-order.
6159
fn get_results(&self) -> &Vec<Self::Value>;

‎src/transform/src/equivalence_propagation.rs

+328-438
Large diffs are not rendered by default.

‎src/transform/tests/testdata/keys

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ Project (#0..=#2, #0..=#2) // { types: "(Int64?, Int64?, Int64?, Int64?, Int64?,
4848
Get x // { types: "(Int64?, Int64?, Int64?)", keys: "([0], [1])" }
4949

5050
====
51-
No change: ThresholdElision, Fixpoint { name: "fixpoint", transforms: [PredicatePushdown { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, NonNullable, ColumnKnowledge { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Demand { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Fixpoint { name: "fixpoint", transforms: [SemijoinIdempotence { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, ReductionPushdown, ReduceElision, LiteralLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, RelationCSE { anf: ANF, normalize_lets: NormalizeLets { inline_mfp: false } }, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: true, strict_join_equivalences: true, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, ProjectionPushdown, NormalizeLets { inline_mfp: false }, Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: true, strict_join_equivalences: true, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, Fixpoint { name: "fixpoint", transforms: [CanonicalizeMfp, ThresholdElision, Join, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, UnionNegateFusion, UnionBranchCancellation, RelationCSE { anf: ANF, normalize_lets: NormalizeLets { inline_mfp: true } }, FoldConstants { limit: Some(10000) }], limit: 100 }, Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: true, strict_join_equivalences: true, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: true, strict_join_equivalences: true, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fixpoint { name: "fixpoint", transforms: [ColumnKnowledge { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }, Demand { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, LiteralLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }], limit: 100 }, LiteralConstraints, Fixpoint { name: "fix_joins", transforms: [JoinImplementation { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }], limit: 100 }, CanonicalizeMfp, RelationCSE { anf: ANF, normalize_lets: NormalizeLets { inline_mfp: false } }, FoldConstants { limit: Some(10000) }, ThresholdElision, NormalizeLets { inline_mfp: false }, Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: true, strict_join_equivalences: false, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }
51+
No change: ThresholdElision, Fixpoint { name: "fixpoint", transforms: [PredicatePushdown { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, EquivalencePropagation, NonNullable, ColumnKnowledge { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Demand { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Fixpoint { name: "fixpoint", transforms: [SemijoinIdempotence { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, ReductionPushdown, ReduceElision, LiteralLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, RelationCSE { anf: ANF, normalize_lets: NormalizeLets { inline_mfp: false } }, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: true, strict_join_equivalences: true, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, ProjectionPushdown, NormalizeLets { inline_mfp: false }, Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: true, strict_join_equivalences: true, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, Fixpoint { name: "fixpoint", transforms: [CanonicalizeMfp, ThresholdElision, Join, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, UnionNegateFusion, UnionBranchCancellation, RelationCSE { anf: ANF, normalize_lets: NormalizeLets { inline_mfp: true } }, FoldConstants { limit: Some(10000) }], limit: 100 }, Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: true, strict_join_equivalences: true, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: true, strict_join_equivalences: true, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fixpoint { name: "fixpoint", transforms: [ColumnKnowledge { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }, Demand { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, LiteralLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }], limit: 100 }, LiteralConstraints, Fixpoint { name: "fix_joins", transforms: [JoinImplementation { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }], limit: 100 }, CanonicalizeMfp, RelationCSE { anf: ANF, normalize_lets: NormalizeLets { inline_mfp: false } }, FoldConstants { limit: Some(10000) }, ThresholdElision, NormalizeLets { inline_mfp: false }, Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: true, strict_join_equivalences: false, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }
5252
====
5353
Final:
5454
Project (#0..=#2, #0..=#2) // { types: "(Int64?, Int64?, Int64?, Int64?, Int64?, Int64?)", keys: "([0], [1])" }

‎src/transform/tests/testdata/steps

+2-2
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ Union
3232
Get x
3333

3434
====
35-
No change: Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: false, strict_join_equivalences: true, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fixpoint { name: "normalize", transforms: [NormalizeLets { inline_mfp: false }, NormalizeOps], limit: 100 }, NonNullRequirements { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fixpoint { name: "fixpoint", transforms: [FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }, ThresholdElision, Fixpoint { name: "fixpoint", transforms: [PredicatePushdown { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, NonNullable, ColumnKnowledge { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Demand { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }
35+
No change: Typecheck { ctx: Mutex { data: {}, poisoned: false, .. }, disallow_new_globals: false, strict_join_equivalences: true, recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fixpoint { name: "normalize", transforms: [NormalizeLets { inline_mfp: false }, NormalizeOps], limit: 100 }, NonNullRequirements { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fixpoint { name: "fixpoint", transforms: [FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }, ThresholdElision, Fixpoint { name: "fixpoint", transforms: [PredicatePushdown { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, EquivalencePropagation, NonNullable, ColumnKnowledge { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Demand { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }
3636
====
3737
Applied Fixpoint { name: "fixpoint", transforms: [SemijoinIdempotence { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, ReductionPushdown, ReduceElision, LiteralLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, RelationCSE { anf: ANF, normalize_lets: NormalizeLets { inline_mfp: false } }, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }:
3838
Return
@@ -103,7 +103,7 @@ Applied Fixpoint { name: "normalize", transforms: [NormalizeLets { inline_mfp: f
103103
====
104104
No change: NonNullRequirements { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fixpoint { name: "fixpoint", transforms: [FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }, ThresholdElision
105105
====
106-
Applied Fixpoint { name: "fixpoint", transforms: [PredicatePushdown { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, NonNullable, ColumnKnowledge { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Demand { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }:
106+
Applied Fixpoint { name: "fixpoint", transforms: [PredicatePushdown { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, EquivalencePropagation, NonNullable, ColumnKnowledge { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Demand { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, Fusion, FlatMapToMap, Join, NormalizeLets { inline_mfp: false }, Reduce, UnionNegateFusion, UnionBranchCancellation, NormalizeLets { inline_mfp: false }, RedundantJoin { recursion_guard: RecursionGuard { depth: RefCell { value: 0 }, limit: 2048 } }, FoldConstants { limit: Some(10000) }] }], limit: 100 }:
107107
(Join [(get u1) (get x)] [[#0 (CallBinary AddInt64 #2 (1 Int64))]] Unimplemented)
108108

109109
====

‎test/sqllogictest/cardinality.slt

+22-22
Original file line numberDiff line numberDiff line change
@@ -298,21 +298,21 @@ Explained Query:
298298
ArrangeBy keys=[[#0]]
299299
Union
300300
Negate
301-
CrossJoin type=differential
302-
ArrangeBy keys=[[]]
303-
Get l1
304-
ArrangeBy keys=[[]]
305-
Distinct project=[]
306-
Project ()
307-
Get l0
308-
Project (#1, #2)
309-
ReadIndex on=person_knows_person person_knows_person_person1id_person2id=[*** full scan ***]
301+
Project (#0, #1)
302+
Join on=(#2 = least(#0, #1) AND #3 = greatest(#0, #1)) type=differential
303+
ArrangeBy keys=[[greatest(#0, #1), least(#0, #1)]]
304+
Get l0
305+
ArrangeBy keys=[[#1, #0]]
306+
Distinct project=[least(#0, #1), greatest(#0, #1)]
307+
Get l1
308+
Get l0
310309
Get l1
311310
cte l1 =
312311
Project (#1, #2)
313-
Get l0
312+
Filter (3 = least(#1, #2)) AND (4 = greatest(#1, #2))
313+
ReadIndex on=person_knows_person person_knows_person_person1id_person2id=[*** full scan ***]
314314
cte l0 =
315-
Filter (3 = least(#1, #2)) AND (4 = greatest(#1, #2))
315+
Project (#1, #2)
316316
ReadIndex on=person_knows_person person_knows_person_person1id_person2id=[*** full scan ***]
317317

318318
Used Indexes:
@@ -412,21 +412,21 @@ Explained Query:
412412
ArrangeBy keys=[[#0]]
413413
Union
414414
Negate
415-
CrossJoin type=differential
416-
ArrangeBy keys=[[]]
417-
Get l1
418-
ArrangeBy keys=[[]]
419-
Distinct project=[]
420-
Project ()
421-
Get l0
422-
Project (#1, #2)
423-
ReadIndex on=person_knows_person person_knows_person_person1id_person2id=[*** full scan ***]
415+
Project (#0, #1)
416+
Join on=(#2 = least(#0, #1) AND #3 = greatest(#0, #1)) type=differential
417+
ArrangeBy keys=[[greatest(#0, #1), least(#0, #1)]]
418+
Get l0
419+
ArrangeBy keys=[[#1, #0]]
420+
Distinct project=[least(#0, #1), greatest(#0, #1)]
421+
Get l1
422+
Get l0
424423
Get l1
425424
cte l1 =
426425
Project (#1, #2)
427-
Get l0
426+
Filter (3 = least(#1, #2)) AND (4 = greatest(#1, #2))
427+
ReadIndex on=person_knows_person person_knows_person_person1id_person2id=[*** full scan ***]
428428
cte l0 =
429-
Filter (3 = least(#1, #2)) AND (4 = greatest(#1, #2))
429+
Project (#1, #2)
430430
ReadIndex on=person_knows_person person_knows_person_person1id_person2id=[*** full scan ***]
431431

432432
Used Indexes:

‎test/sqllogictest/github-9782.slt

+1-1
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ Explained Query:
8787
Project (#1, #2) // { arity: 2 }
8888
Get l0 // { arity: 3 }
8989
Get l1 // { arity: 3 }
90-
Filter (#0 = #1) AND (#0 = #2) // { arity: 3 }
90+
Filter (#0 = #1) // { arity: 3 }
9191
Get l0 // { arity: 3 }
9292
With
9393
cte l1 =

0 commit comments

Comments
 (0)