Auto merge of #117278 - lcnr:try-normalize-ty, r=compiler-errors

new solver normalization improvements

cool beans

At the core of this PR is a `try_normalize_ty` which stops for rigid aliases by using `commit_if_ok`.

Reworks alias-relate to fully normalize both the lhs and rhs and then equate the resulting rigid (or inference) types. This fixes rust-lang/trait-system-refactor-initiative#68 by avoiding the exponential blowup. Also supersedes #116369 by only defining opaque types if the hidden type is rigid.

I removed the stability check in `EvalCtxt::evaluate_goal` due to rust-lang/trait-system-refactor-initiative#75. While I personally have opinions on how to fix it, that still requires further t-types/`@nikomatsakis` buy-in, so I removed that for now. Once we've decided on our approach there, we can revert this commit.

r? `@compiler-errors`

compiler/rustc_middle/src/traits/solve/inspect.rs

@@ -122,6 +122,8 @@ pub enum ProbeStep<'tcx> {
     /// used whenever there are multiple candidates to prove the
     /// current goalby .
     NestedProbe(Probe<'tcx>),
+    CommitIfOkStart,
+    CommitIfOkSuccess,
 }
 
 /// What kind of probe we're in. In case the probe represents a candidate, or
@@ -142,6 +144,9 @@ pub enum ProbeKind<'tcx> {
     /// Used in the probe that wraps normalizing the non-self type for the unsize
     /// trait, which is also structurally matched on.
     UnsizeAssembly,
+    /// A call to `EvalCtxt::commit_if_ok` which failed, causing the work
+    /// to be discarded.
+    CommitIfOk,
     /// During upcasting from some source object to target object type, used to
     /// do a probe to find out what projection type(s) may be used to prove that
     /// the source type upholds all of the target type's object bounds.

compiler/rustc_middle/src/traits/solve/inspect/format.rs

@@ -109,6 +109,9 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> {
             ProbeKind::UpcastProjectionCompatibility => {
                 writeln!(self.f, "PROBING FOR PROJECTION COMPATIBILITY FOR UPCASTING:")
             }
+            ProbeKind::CommitIfOk => {
+                writeln!(self.f, "COMMIT_IF_OK:")
+            }
             ProbeKind::MiscCandidate { name, result } => {
                 writeln!(self.f, "CANDIDATE {name}: {result:?}")
             }
@@ -123,6 +126,8 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> {
                     ProbeStep::AddGoal(goal) => writeln!(this.f, "ADDED GOAL: {goal:?}")?,
                     ProbeStep::EvaluateGoals(eval) => this.format_added_goals_evaluation(eval)?,
                     ProbeStep::NestedProbe(probe) => this.format_probe(probe)?,
+                    ProbeStep::CommitIfOkStart => writeln!(this.f, "COMMIT_IF_OK START")?,
+                    ProbeStep::CommitIfOkSuccess => writeln!(this.f, "COMMIT_IF_OK SUCCESS")?,
                 }
             }
             Ok(())

compiler/rustc_middle/src/ty/sty.rs

@@ -1245,6 +1245,28 @@ impl<'tcx> AliasTy<'tcx> {
         }
     }
 
+    /// Whether this alias type is an opaque.
+    pub fn is_opaque(self, tcx: TyCtxt<'tcx>) -> bool {
+        matches!(self.opt_kind(tcx), Some(ty::AliasKind::Opaque))
+    }
+
+    /// FIXME: rename `AliasTy` to `AliasTerm` and always handle
+    /// constants. This function can then be removed.
+    pub fn opt_kind(self, tcx: TyCtxt<'tcx>) -> Option<ty::AliasKind> {
+        match tcx.def_kind(self.def_id) {
+            DefKind::AssocTy
+                if let DefKind::Impl { of_trait: false } =
+                    tcx.def_kind(tcx.parent(self.def_id)) =>
+            {
+                Some(ty::Inherent)
+            }
+            DefKind::AssocTy => Some(ty::Projection),
+            DefKind::OpaqueTy => Some(ty::Opaque),
+            DefKind::TyAlias => Some(ty::Weak),
+            _ => None,
+        }
+    }
+
     pub fn to_ty(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
         Ty::new_alias(tcx, self.kind(tcx), self)
     }

compiler/rustc_trait_selection/src/solve/alias_relate.rs

@@ -11,18 +11,12 @@
 //! * bidirectional-normalizes-to: If `A` and `B` are both projections, and both
 //!   may apply, then we can compute the "intersection" of both normalizes-to by
 //!   performing them together. This is used specifically to resolve ambiguities.
-use super::{EvalCtxt, SolverMode};
+use super::EvalCtxt;
+use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::traits::query::NoSolution;
 use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
 use rustc_middle::ty;
 
-/// We may need to invert the alias relation direction if dealing an alias on the RHS.
-#[derive(Debug)]
-enum Invert {
-    No,
-    Yes,
-}
-
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     #[instrument(level = "debug", skip(self), ret)]
     pub(super) fn compute_alias_relate_goal(
@@ -31,187 +25,130 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ) -> QueryResult<'tcx> {
         let tcx = self.tcx();
         let Goal { param_env, predicate: (lhs, rhs, direction) } = goal;
-        if lhs.is_infer() || rhs.is_infer() {
-            bug!(
-                "`AliasRelate` goal with an infer var on lhs or rhs which should have been instantiated"
-            );
-        }
-
-        match (lhs.to_alias_ty(tcx), rhs.to_alias_ty(tcx)) {
-            (None, None) => bug!("`AliasRelate` goal without an alias on either lhs or rhs"),
 
-            // RHS is not a projection, only way this is true is if LHS normalizes-to RHS
-            (Some(alias_lhs), None) => self.assemble_normalizes_to_candidate(
-                param_env,
-                alias_lhs,
-                rhs,
-                direction,
-                Invert::No,
-            ),
+        let Some(lhs) = self.try_normalize_term(param_env, lhs)? else {
+            return self.evaluate_added_goals_and_make_canonical_response(Certainty::OVERFLOW);
+        };
 
-            // LHS is not a projection, only way this is true is if RHS normalizes-to LHS
-            (None, Some(alias_rhs)) => self.assemble_normalizes_to_candidate(
-                param_env,
-                alias_rhs,
-                lhs,
-                direction,
-                Invert::Yes,
-            ),
+        let Some(rhs) = self.try_normalize_term(param_env, rhs)? else {
+            return self.evaluate_added_goals_and_make_canonical_response(Certainty::OVERFLOW);
+        };
 
-            (Some(alias_lhs), Some(alias_rhs)) => {
-                debug!("both sides are aliases");
+        let variance = match direction {
+            ty::AliasRelationDirection::Equate => ty::Variance::Invariant,
+            ty::AliasRelationDirection::Subtype => ty::Variance::Covariant,
+        };
 
-                let mut candidates = Vec::new();
-                // LHS normalizes-to RHS
-                candidates.extend(self.assemble_normalizes_to_candidate(
-                    param_env,
-                    alias_lhs,
-                    rhs,
-                    direction,
-                    Invert::No,
-                ));
-                // RHS normalizes-to RHS
-                candidates.extend(self.assemble_normalizes_to_candidate(
-                    param_env,
-                    alias_rhs,
-                    lhs,
-                    direction,
-                    Invert::Yes,
-                ));
-                // Relate via args
-                candidates.extend(
-                    self.assemble_subst_relate_candidate(
-                        param_env, alias_lhs, alias_rhs, direction,
-                    ),
-                );
-                debug!(?candidates);
+        match (lhs.to_alias_ty(tcx), rhs.to_alias_ty(tcx)) {
+            (None, None) => {
+                self.relate(param_env, lhs, variance, rhs)?;
+                self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+            }
 
-                if let Some(merged) = self.try_merge_responses(&candidates) {
-                    Ok(merged)
+            (Some(alias), None) => {
+                if rhs.is_infer() {
+                    self.relate(param_env, lhs, variance, rhs)?;
+                    self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+                } else if alias.is_opaque(tcx) {
+                    self.define_opaque(param_env, alias, rhs)
                 } else {
-                    // When relating two aliases and we have ambiguity, if both
-                    // aliases can be normalized to something, we prefer
-                    // "bidirectionally normalizing" both of them within the same
-                    // candidate.
-                    //
-                    // See <https://github.com/rust-lang/trait-system-refactor-initiative/issues/25>.
-                    //
-                    // As this is incomplete, we must not do so during coherence.
-                    match self.solver_mode() {
-                        SolverMode::Normal => {
-                            if let Ok(bidirectional_normalizes_to_response) = self
-                                .assemble_bidirectional_normalizes_to_candidate(
-                                    param_env, lhs, rhs, direction,
-                                )
-                            {
-                                Ok(bidirectional_normalizes_to_response)
-                            } else {
-                                self.flounder(&candidates)
-                            }
-                        }
-                        SolverMode::Coherence => self.flounder(&candidates),
-                    }
+                    Err(NoSolution)
                 }
             }
+            (None, Some(alias)) => {
+                if lhs.is_infer() {
+                    self.relate(param_env, lhs, variance, rhs)?;
+                    self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+                } else if alias.is_opaque(tcx) {
+                    self.define_opaque(param_env, alias, lhs)
+                } else {
+                    Err(NoSolution)
+                }
+            }
+
+            (Some(alias_lhs), Some(alias_rhs)) => {
+                self.relate_rigid_alias_or_opaque(param_env, alias_lhs, variance, alias_rhs)
+            }
         }
     }
 
-    #[instrument(level = "debug", skip(self), ret)]
-    fn assemble_normalizes_to_candidate(
+    /// Normalize the `term` to equate it later. This does not define opaque types.
+    #[instrument(level = "debug", skip(self, param_env), ret)]
+    fn try_normalize_term(
         &mut self,
         param_env: ty::ParamEnv<'tcx>,
-        alias: ty::AliasTy<'tcx>,
-        other: ty::Term<'tcx>,
-        direction: ty::AliasRelationDirection,
-        invert: Invert,
-    ) -> QueryResult<'tcx> {
-        self.probe_misc_candidate("normalizes-to").enter(|ecx| {
-            ecx.normalizes_to_inner(param_env, alias, other, direction, invert)?;
-            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-        })
+        term: ty::Term<'tcx>,
+    ) -> Result<Option<ty::Term<'tcx>>, NoSolution> {
+        match term.unpack() {
+            ty::TermKind::Ty(ty) => {
+                // We do no define opaque types here but instead do so in `relate_rigid_alias_or_opaque`.
+                Ok(self
+                    .try_normalize_ty_recur(param_env, DefineOpaqueTypes::No, 0, ty)
+                    .map(Into::into))
+            }
+            ty::TermKind::Const(_) => {
+                if let Some(alias) = term.to_alias_ty(self.tcx()) {
+                    let term = self.next_term_infer_of_kind(term);
+                    self.add_goal(Goal::new(
+                        self.tcx(),
+                        param_env,
+                        ty::ProjectionPredicate { projection_ty: alias, term },
+                    ));
+                    self.try_evaluate_added_goals()?;
+                    Ok(Some(self.resolve_vars_if_possible(term)))
+                } else {
+                    Ok(Some(term))
+                }
+            }
+        }
     }
 
-    // Computes the normalizes-to branch, with side-effects. This must be performed
-    // in a probe in order to not taint the evaluation context.
-    fn normalizes_to_inner(
+    fn define_opaque(
         &mut self,
         param_env: ty::ParamEnv<'tcx>,
-        alias: ty::AliasTy<'tcx>,
-        other: ty::Term<'tcx>,
-        direction: ty::AliasRelationDirection,
-        invert: Invert,
-    ) -> Result<(), NoSolution> {
-        let other = match direction {
-            // This is purely an optimization. No need to instantiate a new
-            // infer var and equate the RHS to it.
-            ty::AliasRelationDirection::Equate => other,
-
-            // Instantiate an infer var and subtype our RHS to it, so that we
-            // properly represent a subtype relation between the LHS and RHS
-            // of the goal.
-            ty::AliasRelationDirection::Subtype => {
-                let fresh = self.next_term_infer_of_kind(other);
-                let (sub, sup) = match invert {
-                    Invert::No => (fresh, other),
-                    Invert::Yes => (other, fresh),
-                };
-                self.sub(param_env, sub, sup)?;
-                fresh
-            }
-        };
+        opaque: ty::AliasTy<'tcx>,
+        term: ty::Term<'tcx>,
+    ) -> QueryResult<'tcx> {
         self.add_goal(Goal::new(
             self.tcx(),
             param_env,
-            ty::ProjectionPredicate { projection_ty: alias, term: other },
+            ty::ProjectionPredicate { projection_ty: opaque, term },
         ));
-
-        Ok(())
+        self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
-    fn assemble_subst_relate_candidate(
+    fn relate_rigid_alias_or_opaque(
         &mut self,
         param_env: ty::ParamEnv<'tcx>,
-        alias_lhs: ty::AliasTy<'tcx>,
-        alias_rhs: ty::AliasTy<'tcx>,
-        direction: ty::AliasRelationDirection,
+        lhs: ty::AliasTy<'tcx>,
+        variance: ty::Variance,
+        rhs: ty::AliasTy<'tcx>,
     ) -> QueryResult<'tcx> {
-        self.probe_misc_candidate("args relate").enter(|ecx| {
-            match direction {
-                ty::AliasRelationDirection::Equate => {
-                    ecx.eq(param_env, alias_lhs, alias_rhs)?;
-                }
-                ty::AliasRelationDirection::Subtype => {
-                    ecx.sub(param_env, alias_lhs, alias_rhs)?;
-                }
-            }
+        let tcx = self.tcx();
+        let mut candidates = vec![];
+        if lhs.is_opaque(tcx) {
+            candidates.extend(
+                self.probe_misc_candidate("define-lhs-opaque")
+                    .enter(|ecx| ecx.define_opaque(param_env, lhs, rhs.to_ty(tcx).into())),
+            );
+        }
 
-            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-        })
-    }
+        if rhs.is_opaque(tcx) {
+            candidates.extend(
+                self.probe_misc_candidate("define-rhs-opaque")
+                    .enter(|ecx| ecx.define_opaque(param_env, rhs, lhs.to_ty(tcx).into())),
+            );
+        }
 
-    fn assemble_bidirectional_normalizes_to_candidate(
-        &mut self,
-        param_env: ty::ParamEnv<'tcx>,
-        lhs: ty::Term<'tcx>,
-        rhs: ty::Term<'tcx>,
-        direction: ty::AliasRelationDirection,
-    ) -> QueryResult<'tcx> {
-        self.probe_misc_candidate("bidir normalizes-to").enter(|ecx| {
-            ecx.normalizes_to_inner(
-                param_env,
-                lhs.to_alias_ty(ecx.tcx()).unwrap(),
-                rhs,
-                direction,
-                Invert::No,
-            )?;
-            ecx.normalizes_to_inner(
-                param_env,
-                rhs.to_alias_ty(ecx.tcx()).unwrap(),
-                lhs,
-                direction,
-                Invert::Yes,
-            )?;
+        candidates.extend(self.probe_misc_candidate("args-relate").enter(|ecx| {
+            ecx.relate(param_env, lhs, variance, rhs)?;
             ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-        })
+        }));
+
+        if let Some(result) = self.try_merge_responses(&candidates) {
+            Ok(result)
+        } else {
+            self.flounder(&candidates)
+        }
     }
 }