Merge pull request #24155 from frankmcsherry/equivalence_propagation

Equivalence propagation

Author: Alexander Alexandrov

misc/python/materialize/mzcompose/__init__.py

@@ -65,6 +65,7 @@
     "enable_comment": "true",
     "enable_disk_cluster_replicas": "true",
     "enable_eager_delta_joins": "true",
+    "enable_equivalence_propagation": "true",
     "enable_expressions_in_limit_syntax": "true",
     "enable_logical_compaction_window": "true",
     "enable_multi_worker_storage_persist_sink": "true",

src/adapter/src/optimize/mod.rs

@@ -155,35 +155,28 @@ where
 ///
 /// To add a new feature flag, do the following steps:
 ///
-/// 1. To make the flag available to all stages in our [`Optimize`] pipelines:
-///    1. Add the flag as an [`OptimizerConfig`] field.
-///
-/// 2. To allow engineers to set a system-wide override for this feature flag:
-///    1. Add the flag to the `feature_flags!(...)` macro call.
-///    2. Extend the `From<&SystemVars>` implementation for [`OptimizerConfig`].
+/// 1. To make the flag available to all stages in our [`Optimize`] pipelines
+///    and allow engineers to set a system-wide override:
+///    1. Add the flag to the `optimizer_feature_flags!(...)` macro call.
+///    2. Add the flag to the `feature_flags!(...)` macro call and extend the
+///       `From<&SystemVars>` implementation for [`OptimizerFeatures`].
 ///
-/// 3. To enable `EXPLAIN ... WITH(...)` overrides which will allow engineers to
+/// 2. To enable `EXPLAIN ... WITH(...)` overrides which will allow engineers to
 ///    inspect plan differences before deploying the optimizer changes:
-///    1. Add the flag as a [`mz_repr::explain::ExplainConfig`] field.
-///    2. Add the flag to the `ExplainPlanOptionName` definition.
-///    3. Add the flag to the `generate_extracted_config!(ExplainPlanOption,
+///    1. Add the flag to the `ExplainPlanOptionName` definition.
+///    2. Add the flag to the `generate_extracted_config!(ExplainPlanOption,
 ///       ...)` macro call.
-///    4. Extend the `TryFrom<ExplainPlanOptionExtracted>` implementation for
+///    3. Extend the `TryFrom<ExplainPlanOptionExtracted>` implementation for
 ///       [`mz_repr::explain::ExplainConfig`].
-///    5. Extend the `OverrideFrom<ExplainContext>` implementation for
-///       [`OptimizerConfig`].
 ///
-/// 4. To enable `CLUSTER ... FEATURES(...)` overrides which will allow
+/// 3. To enable `CLUSTER ... FEATURES(...)` overrides which will allow
 ///    engineers to experiment with runtime differences before deploying the
 ///    optimizer changes:
-///    1. Add the flag to the `optimizer_feature_flags!(...)` macro call.
-///    2. Add the flag to the `ClusterFeatureName` definition.
-///    3. Add the flag to the `generate_extracted_config!(ClusterFeature, ...)`
+///    1. Add the flag to the `ClusterFeatureName` definition.
+///    2. Add the flag to the `generate_extracted_config!(ClusterFeature, ...)`
 ///       macro call.
-///    4. Extend the `let optimizer_feature_overrides = ...` call in
+///    3. Extend the `let optimizer_feature_overrides = ...` call in
 ///       `plan_create_cluster`.
-///    4. Extend the `OverrideFrom<OptimizerFeatureOverrides>` implementation
-///       for [`OptimizerConfig`].
 #[derive(Clone, Debug)]
 pub struct OptimizerConfig {
     /// The mode in which the optimizer runs.

src/repr/src/optimize.rs

@@ -97,6 +97,8 @@ optimizer_feature_flags!({
     enable_consolidate_after_union_negate: bool,
     // Bound from `SystemVars::enable_eager_delta_joins`.
     enable_eager_delta_joins: bool,
+    // Enable the `EquivalencePropagation` transform in the optimizer.
+    enable_equivalence_propagation: bool,
     // Bound from `SystemVars::enable_new_outer_join_lowering`.
     enable_new_outer_join_lowering: bool,
     // Bound from `SystemVars::enable_reduce_mfp_fusion`.

src/sql-lexer/src/keywords.txt

@@ -138,6 +138,7 @@ End
 Endpoint
 Enforced
 Envelope
+Equivalence
 Error
 Escape
 Every
@@ -310,6 +311,7 @@ Primary
 Privatelink
 Privileges
 Progress
+Propagation
 Protobuf
 Protocol
 Publication

src/sql-parser/src/ast/defs/statement.rs

@@ -1732,6 +1732,7 @@ pub enum ClusterFeatureName {
     ReoptimizeImportedViews,
     EnableNewOuterJoinLowering,
     EnableEagerDeltaJoins,
+    EnableEquivalencePropagation,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
@@ -3093,6 +3094,7 @@ pub enum ExplainPlanOptionName {
     ReoptimizeImportedViews,
     EnableNewOuterJoinLowering,
     EnableEagerDeltaJoins,
+    EnableEquivalencePropagation,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]

src/sql/src/plan/statement/ddl.rs

@@ -3336,7 +3336,8 @@ generate_extracted_config!(
     ClusterFeature,
     (ReoptimizeImportedViews, Option<bool>, Default(None)),
     (EnableEagerDeltaJoins, Option<bool>, Default(None)),
-    (EnableNewOuterJoinLowering, Option<bool>, Default(None))
+    (EnableNewOuterJoinLowering, Option<bool>, Default(None)),
+    (EnableEquivalencePropagation, Option<bool>, Default(None))
 );
 
 pub fn plan_create_cluster(
@@ -3413,12 +3414,14 @@ pub fn plan_create_cluster(
             reoptimize_imported_views,
             enable_eager_delta_joins,
             enable_new_outer_join_lowering,
+            enable_equivalence_propagation,
             seen: _,
         } = ClusterFeatureExtracted::try_from(features)?;
         let optimizer_feature_overrides = OptimizerFeatureOverrides {
             reoptimize_imported_views,
             enable_eager_delta_joins,
             enable_new_outer_join_lowering,
+            enable_equivalence_propagation,
             ..Default::default()
         };
 

src/sql/src/plan/statement/dml.rs

@@ -353,7 +353,8 @@ generate_extracted_config!(
     (Types, bool, Default(false)),
     (ReoptimizeImportedViews, Option<bool>, Default(None)),
     (EnableNewOuterJoinLowering, Option<bool>, Default(None)),
-    (EnableEagerDeltaJoins, Option<bool>, Default(None))
+    (EnableEagerDeltaJoins, Option<bool>, Default(None)),
+    (EnableEquivalencePropagation, Option<bool>, Default(None))
 );
 
 impl TryFrom<ExplainPlanOptionExtracted> for ExplainConfig {
@@ -395,6 +396,7 @@ impl TryFrom<ExplainPlanOptionExtracted> for ExplainConfig {
             types: v.types,
             features: OptimizerFeatureOverrides {
                 enable_eager_delta_joins: v.enable_eager_delta_joins,
+                enable_equivalence_propagation: v.enable_equivalence_propagation,
                 enable_new_outer_join_lowering: v.enable_new_outer_join_lowering,
                 reoptimize_imported_views: v.reoptimize_imported_views,
                 ..Default::default()

src/sql/src/session/vars/definitions.rs

@@ -2105,13 +2105,21 @@ feature_flags!(
         internal: true,
         enable_for_item_parsing: false,
     },
+    {
+        name: enable_equivalence_propagation,
+        desc: "Enable the EquivalencePropagation transform in the optimizer",
+        default: true, // TODO(aalexandrov): revert this to false
+        internal: true,
+        enable_for_item_parsing: false,
+    },
 );
 
 impl From<&super::SystemVars> for OptimizerFeatures {
     fn from(vars: &super::SystemVars) -> Self {
         Self {
             enable_consolidate_after_union_negate: vars.enable_consolidate_after_union_negate(),
             enable_eager_delta_joins: vars.enable_eager_delta_joins(),
+            enable_equivalence_propagation: vars.enable_equivalence_propagation(),
             enable_new_outer_join_lowering: vars.enable_new_outer_join_lowering(),
             enable_reduce_mfp_fusion: vars.enable_reduce_mfp_fusion(),
             persist_fast_path_limit: vars.persist_fast_path_limit(),

src/transform/src/analysis.rs

@@ -9,6 +9,8 @@
 
 //! Traits and types for reusable expression analysis
 
+pub mod equivalences;
+
 use mz_expr::MirRelationExpr;
 
 pub use common::{Derived, DerivedBuilder, DerivedView};
@@ -121,6 +123,7 @@ pub mod common {
     ///
     /// This is best thought of as a node in a tree rather
     #[allow(missing_debug_implementations)]
+    #[derive(Copy, Clone)]
     pub struct DerivedView<'a> {
         derived: &'a Derived,
         lower: usize,