diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs index 3b40882ef572d..2833b6980e2de 100644 --- a/compiler/rustc_trait_selection/src/traits/coherence.rs +++ b/compiler/rustc_trait_selection/src/traits/coherence.rs @@ -100,7 +100,10 @@ pub fn overlapping_impls( let impl1_ref = tcx.impl_trait_ref(impl1_def_id); let impl2_ref = tcx.impl_trait_ref(impl2_def_id); let may_overlap = match (impl1_ref, impl2_ref) { - (Some(a), Some(b)) => drcx.args_may_unify(a.skip_binder().args, b.skip_binder().args), + (Some(a), Some(b)) => { + // This call site can be hot. + drcx.inlined_args_may_unify(a.skip_binder().args, b.skip_binder().args) + } (None, None) => { let self_ty1 = tcx.type_of(impl1_def_id).skip_binder(); let self_ty2 = tcx.type_of(impl2_def_id).skip_binder(); diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs index 11b6b826efe54..5ab6274a16e52 100644 --- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs +++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs @@ -561,9 +561,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { // consider a "quick reject". This avoids creating more types // and so forth that we need to. let impl_trait_header = self.tcx().impl_trait_header(impl_def_id).unwrap(); - if !drcx - .args_may_unify(obligation_args, impl_trait_header.trait_ref.skip_binder().args) - { + // This call site can be hot. + if !drcx.inlined_args_may_unify( + obligation_args, + impl_trait_header.trait_ref.skip_binder().args, + ) { return; } diff --git a/compiler/rustc_type_ir/src/fast_reject.rs b/compiler/rustc_type_ir/src/fast_reject.rs index 9955e92b55a5c..71afd29cc8c23 100644 --- a/compiler/rustc_type_ir/src/fast_reject.rs +++ b/compiler/rustc_type_ir/src/fast_reject.rs @@ -220,6 +220,15 @@ impl bool { + self.inlined_args_may_unify_inner(obligation_args, impl_args, Self::STARTING_DEPTH) + } + pub fn args_may_unify( self, obligation_args: I::GenericArgs, @@ -228,8 +237,23 @@ impl bool { - self.types_may_unify_inner(lhs, rhs, Self::STARTING_DEPTH) + #[inline(always)] + fn inlined_arg_may_unify(self, obl: I::GenericArg, imp: I::GenericArg, depth: usize) -> bool { + match (obl.kind(), imp.kind()) { + // We don't fast reject based on regions. + (ty::GenericArgKind::Lifetime(_), ty::GenericArgKind::Lifetime(_)) => true, + (ty::GenericArgKind::Type(obl), ty::GenericArgKind::Type(imp)) => { + self.types_may_unify_inner(obl, imp, depth) + } + (ty::GenericArgKind::Const(obl), ty::GenericArgKind::Const(imp)) => { + self.consts_may_unify_inner(obl, imp) + } + _ => panic!("kind mismatch: {obl:?} {imp:?}"), + } + } + + fn arg_may_unify(self, obl: I::GenericArg, imp: I::GenericArg, depth: usize) -> bool { + self.inlined_arg_may_unify(obl, imp, depth) } fn args_may_unify_inner( @@ -237,23 +261,35 @@ impl bool { + iter::zip(obligation_args.iter(), impl_args.iter()) + .all(|(obl, imp)| self.arg_may_unify(obl, imp, depth)) + } + + #[inline(always)] + fn inlined_args_may_unify_inner( + self, + obligation_args: I::GenericArgs, + impl_args: I::GenericArgs, + depth: usize, ) -> bool { // No need to decrement the depth here as this function is only // recursively reachable via `types_may_unify_inner` which already // increments the depth for us. - iter::zip(obligation_args.iter(), impl_args.iter()).all(|(obl, imp)| { - match (obl.kind(), imp.kind()) { - // We don't fast reject based on regions. - (ty::GenericArgKind::Lifetime(_), ty::GenericArgKind::Lifetime(_)) => true, - (ty::GenericArgKind::Type(obl), ty::GenericArgKind::Type(imp)) => { - self.types_may_unify_inner(obl, imp, depth) - } - (ty::GenericArgKind::Const(obl), ty::GenericArgKind::Const(imp)) => { - self.consts_may_unify_inner(obl, imp) - } - _ => panic!("kind mismatch: {obl:?} {imp:?}"), - } - }) + + // The len==1 case accounts for 99.9% of occurrences in the benchmarks + // where this code is hot, e.g. `bitmaps-3.1.0` and `typenum-1.17.0`. + if obligation_args.len() == 1 { + let obl = obligation_args.as_slice()[0]; + let imp = impl_args.as_slice()[0]; + self.inlined_arg_may_unify(obl, imp, depth) + } else { + self.args_may_unify_inner(obligation_args, impl_args, depth) + } + } + + pub fn types_may_unify(self, lhs: I::Ty, rhs: I::Ty) -> bool { + self.types_may_unify_inner(lhs, rhs, Self::STARTING_DEPTH) } fn types_may_unify_inner(self, lhs: I::Ty, rhs: I::Ty, depth: usize) -> bool { @@ -320,7 +356,9 @@ impl match rhs.kind() { ty::Adt(rhs_def, rhs_args) => { - lhs_def == rhs_def && self.args_may_unify_inner(lhs_args, rhs_args, depth) + // This call site can be hot. + lhs_def == rhs_def + && self.inlined_args_may_unify_inner(lhs_args, rhs_args, depth) } _ => false, },