Skip to content

Commit 30f168e

Browse files
borsbors
committed
Auto merge of #137907 - compiler-errors:inline-fnonce, r=saethlin
Inline `FnOnce`/`FnMut`/`Fn` shims once again This PR fixes the argument checking for `extern "rust-call"` ABI functions with a spread arg, which do no expect their arguments to be exploded from a tuple like closures do. Secondly, it removes the hack that prevented them from being inlined. This results in more work done by the compiler, but it does end up allowing us to inline functions we didn't before. Fixes #137901
2 parents 07b5eee + d33946c commit 30f168e

23 files changed

+397
-123
lines changed

‎compiler/rustc_mir_transform/src/inline.rs

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -606,14 +606,14 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>(
606606
ty::EarlyBinder::bind(callee_body.clone()),
607607
) else {
608608
debug!("failed to normalize callee body");
609-
return Err("implementation limitation");
609+
return Err("implementation limitation -- could not normalize callee body");
610610
};
611611

612612
// Normally, this shouldn't be required, but trait normalization failure can create a
613613
// validation ICE.
614614
if !validate_types(tcx, inliner.typing_env(), &callee_body, &caller_body).is_empty() {
615615
debug!("failed to validate callee body");
616-
return Err("implementation limitation");
616+
return Err("implementation limitation -- callee body failed validation");
617617
}
618618

619619
// Check call signature compatibility.
@@ -622,17 +622,9 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>(
622622
let output_type = callee_body.return_ty();
623623
if !util::sub_types(tcx, inliner.typing_env(), output_type, destination_ty) {
624624
trace!(?output_type, ?destination_ty);
625-
debug!("failed to normalize return type");
626-
return Err("implementation limitation");
625+
return Err("implementation limitation -- return type mismatch");
627626
}
628627
if callsite.fn_sig.abi() == ExternAbi::RustCall {
629-
// FIXME: Don't inline user-written `extern "rust-call"` functions,
630-
// since this is generally perf-negative on rustc, and we hope that
631-
// LLVM will inline these functions instead.
632-
if callee_body.spread_arg.is_some() {
633-
return Err("user-written rust-call functions");
634-
}
635-
636628
let (self_arg, arg_tuple) = match &args[..] {
637629
[arg_tuple] => (None, arg_tuple),
638630
[self_arg, arg_tuple] => (Some(self_arg), arg_tuple),
@@ -642,12 +634,17 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>(
642634
let self_arg_ty = self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, tcx));
643635

644636
let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, tcx);
645-
let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else {
646-
bug!("Closure arguments are not passed as a tuple");
637+
let arg_tys = if callee_body.spread_arg.is_some() {
638+
std::slice::from_ref(&arg_tuple_ty)
639+
} else {
640+
let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else {
641+
bug!("Closure arguments are not passed as a tuple");
642+
};
643+
arg_tuple_tys.as_slice()
647644
};
648645

649646
for (arg_ty, input) in
650-
self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter())
647+
self_arg_ty.into_iter().chain(arg_tys.iter().copied()).zip(callee_body.args_iter())
651648
{
652649
let input_type = callee_body.local_decls[input].ty;
653650
if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
@@ -663,7 +660,7 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>(
663660
if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
664661
trace!(?arg_ty, ?input_type);
665662
debug!("failed to normalize argument type");
666-
return Err("implementation limitation");
663+
return Err("implementation limitation -- arg mismatch");
667664
}
668665
}
669666
}
@@ -693,13 +690,13 @@ fn check_mir_is_available<'tcx, I: Inliner<'tcx>>(
693690
// won't cause cycles on this.
694691
if !inliner.tcx().is_mir_available(callee_def_id) {
695692
debug!("item MIR unavailable");
696-
return Err("implementation limitation");
693+
return Err("implementation limitation -- MIR unavailable");
697694
}
698695
}
699696
// These have no own callable MIR.
700697
InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => {
701698
debug!("instance without MIR (intrinsic / virtual)");
702-
return Err("implementation limitation");
699+
return Err("implementation limitation -- cannot inline intrinsic");
703700
}
704701

705702
// FIXME(#127030): `ConstParamHasTy` has bad interactions with
@@ -709,7 +706,7 @@ fn check_mir_is_available<'tcx, I: Inliner<'tcx>>(
709706
// substituted.
710707
InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => {
711708
debug!("still needs substitution");
712-
return Err("implementation limitation");
709+
return Err("implementation limitation -- HACK for dropping polymorphic type");
713710
}
714711

715712
// This cannot result in an immediate cycle since the callee MIR is a shim, which does
@@ -1060,8 +1057,7 @@ fn make_call_args<'tcx, I: Inliner<'tcx>>(
10601057

10611058
closure_ref_arg.chain(tuple_tmp_args).collect()
10621059
} else {
1063-
// FIXME(edition_2024): switch back to a normal method call.
1064-
<_>::into_iter(args)
1060+
args.into_iter()
10651061
.map(|a| create_temp_if_necessary(inliner, a.node, callsite, caller_body, return_block))
10661062
.collect()
10671063
}

‎tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,42 @@
77
let mut _0: ();
88
let mut _3: &mut std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
99
let mut _4: I;
10+
+ scope 1 (inlined <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut) {
11+
+ let mut _5: &mut dyn std::ops::FnMut<I, Output = ()>;
12+
+ let mut _6: std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
13+
+ let mut _7: *const dyn std::ops::FnMut<I, Output = ()>;
14+
+ }
1015

1116
bb0: {
1217
StorageLive(_3);
1318
_3 = &mut _1;
1419
StorageLive(_4);
1520
_4 = move _2;
16-
_0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable];
21+
- _0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable];
22+
+ StorageLive(_6);
23+
+ StorageLive(_7);
24+
+ StorageLive(_5);
25+
+ _6 = copy (*_3);
26+
+ _7 = copy ((_6.0: std::ptr::Unique<dyn std::ops::FnMut<I, Output = ()>>).0: std::ptr::NonNull<dyn std::ops::FnMut<I, Output = ()>>) as *const dyn std::ops::FnMut<I, Output = ()> (Transmute);
27+
+ _5 = &mut (*_7);
28+
+ _0 = <dyn FnMut<I, Output = ()> as FnMut<I>>::call_mut(move _5, move _4) -> [return: bb2, unwind unreachable];
1729
}
1830

1931
bb1: {
20-
StorageDead(_4);
21-
StorageDead(_3);
22-
drop(_1) -> [return: bb2, unwind unreachable];
32+
- StorageDead(_4);
33+
- StorageDead(_3);
34+
- drop(_1) -> [return: bb2, unwind unreachable];
35+
+ return;
2336
}
2437

2538
bb2: {
26-
return;
39+
- return;
40+
+ StorageDead(_5);
41+
+ StorageDead(_7);
42+
+ StorageDead(_6);
43+
+ StorageDead(_4);
44+
+ StorageDead(_3);
45+
+ drop(_1) -> [return: bb1, unwind unreachable];
2746
}
2847
}
2948

‎tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,31 +7,54 @@
77
let mut _0: ();
88
let mut _3: &mut std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
99
let mut _4: I;
10+
+ scope 1 (inlined <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut) {
11+
+ let mut _5: &mut dyn std::ops::FnMut<I, Output = ()>;
12+
+ let mut _6: std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
13+
+ let mut _7: *const dyn std::ops::FnMut<I, Output = ()>;
14+
+ }
1015

1116
bb0: {
1217
StorageLive(_3);
1318
_3 = &mut _1;
1419
StorageLive(_4);
1520
_4 = move _2;
16-
_0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind: bb3];
21+
- _0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind: bb3];
22+
+ StorageLive(_6);
23+
+ StorageLive(_7);
24+
+ StorageLive(_5);
25+
+ _6 = copy (*_3);
26+
+ _7 = copy ((_6.0: std::ptr::Unique<dyn std::ops::FnMut<I, Output = ()>>).0: std::ptr::NonNull<dyn std::ops::FnMut<I, Output = ()>>) as *const dyn std::ops::FnMut<I, Output = ()> (Transmute);
27+
+ _5 = &mut (*_7);
28+
+ _0 = <dyn FnMut<I, Output = ()> as FnMut<I>>::call_mut(move _5, move _4) -> [return: bb4, unwind: bb2];
1729
}
1830

1931
bb1: {
20-
StorageDead(_4);
21-
StorageDead(_3);
22-
drop(_1) -> [return: bb2, unwind: bb4];
32+
- StorageDead(_4);
33+
- StorageDead(_3);
34+
- drop(_1) -> [return: bb2, unwind: bb4];
35+
+ return;
2336
}
2437

25-
bb2: {
26-
return;
38+
- bb2: {
39+
- return;
40+
+ bb2 (cleanup): {
41+
+ drop(_1) -> [return: bb3, unwind terminate(cleanup)];
2742
}
2843

2944
bb3 (cleanup): {
30-
drop(_1) -> [return: bb4, unwind terminate(cleanup)];
45+
- drop(_1) -> [return: bb4, unwind terminate(cleanup)];
46+
+ resume;
3147
}
3248

33-
bb4 (cleanup): {
34-
resume;
49+
- bb4 (cleanup): {
50+
- resume;
51+
+ bb4: {
52+
+ StorageDead(_5);
53+
+ StorageDead(_7);
54+
+ StorageDead(_6);
55+
+ StorageDead(_4);
56+
+ StorageDead(_3);
57+
+ drop(_1) -> [return: bb1, unwind: bb3];
3558
}
3659
}
3760

‎tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,6 @@ use std::marker::Tuple;
88
// EMIT_MIR dont_ice_on_generic_rust_call.call.Inline.diff
99
pub fn call<I: Tuple>(mut mock: Box<dyn FnMut<I, Output = ()>>, input: I) {
1010
// CHECK-LABEL: fn call(
11-
// CHECK-NOT: inlined
11+
// CHECK: (inlined <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut)
1212
mock.call_mut(input)
1313
}

‎tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,26 +7,46 @@
77
let _2: ();
88
let mut _3: &std::boxed::Box<dyn std::ops::Fn(i32)>;
99
let mut _4: (i32,);
10+
+ scope 1 (inlined <Box<dyn Fn(i32)> as Fn<(i32,)>>::call) {
11+
+ let mut _5: &dyn std::ops::Fn(i32);
12+
+ let mut _6: std::boxed::Box<dyn std::ops::Fn(i32)>;
13+
+ let mut _7: *const dyn std::ops::Fn(i32);
14+
+ }
1015

1116
bb0: {
1217
StorageLive(_2);
1318
StorageLive(_3);
1419
_3 = &_1;
1520
StorageLive(_4);
1621
_4 = (const 1_i32,);
17-
_2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable];
22+
- _2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable];
23+
+ StorageLive(_6);
24+
+ StorageLive(_7);
25+
+ StorageLive(_5);
26+
+ _6 = copy (*_3);
27+
+ _7 = copy ((_6.0: std::ptr::Unique<dyn std::ops::Fn(i32)>).0: std::ptr::NonNull<dyn std::ops::Fn(i32)>) as *const dyn std::ops::Fn(i32) (Transmute);
28+
+ _5 = &(*_7);
29+
+ _2 = <dyn Fn(i32) as Fn<(i32,)>>::call(move _5, move _4) -> [return: bb2, unwind unreachable];
1830
}
1931

2032
bb1: {
33+
+ return;
34+
+ }
35+
+
36+
+ bb2: {
37+
+ StorageDead(_5);
38+
+ StorageDead(_7);
39+
+ StorageDead(_6);
2140
StorageDead(_4);
2241
StorageDead(_3);
2342
StorageDead(_2);
2443
_0 = const ();
25-
drop(_1) -> [return: bb2, unwind unreachable];
26-
}
27-
28-
bb2: {
29-
return;
44+
- drop(_1) -> [return: bb2, unwind unreachable];
45+
- }
46+
-
47+
- bb2: {
48+
- return;
49+
+ drop(_1) -> [return: bb1, unwind unreachable];
3050
}
3151
}
3252

‎tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,34 +7,59 @@
77
let _2: ();
88
let mut _3: &std::boxed::Box<dyn std::ops::Fn(i32)>;
99
let mut _4: (i32,);
10+
+ scope 1 (inlined <Box<dyn Fn(i32)> as Fn<(i32,)>>::call) {
11+
+ let mut _5: &dyn std::ops::Fn(i32);
12+
+ let mut _6: std::boxed::Box<dyn std::ops::Fn(i32)>;
13+
+ let mut _7: *const dyn std::ops::Fn(i32);
14+
+ }
1015

1116
bb0: {
1217
StorageLive(_2);
1318
StorageLive(_3);
1419
_3 = &_1;
1520
StorageLive(_4);
1621
_4 = (const 1_i32,);
17-
_2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind: bb3];
22+
- _2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind: bb3];
23+
+ StorageLive(_6);
24+
+ StorageLive(_7);
25+
+ StorageLive(_5);
26+
+ _6 = copy (*_3);
27+
+ _7 = copy ((_6.0: std::ptr::Unique<dyn std::ops::Fn(i32)>).0: std::ptr::NonNull<dyn std::ops::Fn(i32)>) as *const dyn std::ops::Fn(i32) (Transmute);
28+
+ _5 = &(*_7);
29+
+ _2 = <dyn Fn(i32) as Fn<(i32,)>>::call(move _5, move _4) -> [return: bb4, unwind: bb2];
1830
}
1931

2032
bb1: {
21-
StorageDead(_4);
22-
StorageDead(_3);
23-
StorageDead(_2);
24-
_0 = const ();
25-
drop(_1) -> [return: bb2, unwind: bb4];
33+
- StorageDead(_4);
34+
- StorageDead(_3);
35+
- StorageDead(_2);
36+
- _0 = const ();
37+
- drop(_1) -> [return: bb2, unwind: bb4];
38+
+ return;
2639
}
2740

28-
bb2: {
29-
return;
41+
- bb2: {
42+
- return;
43+
+ bb2 (cleanup): {
44+
+ drop(_1) -> [return: bb3, unwind terminate(cleanup)];
3045
}
3146

3247
bb3 (cleanup): {
33-
drop(_1) -> [return: bb4, unwind terminate(cleanup)];
48+
- drop(_1) -> [return: bb4, unwind terminate(cleanup)];
49+
+ resume;
3450
}
3551

36-
bb4 (cleanup): {
37-
resume;
52+
- bb4 (cleanup): {
53+
- resume;
54+
+ bb4: {
55+
+ StorageDead(_5);
56+
+ StorageDead(_7);
57+
+ StorageDead(_6);
58+
+ StorageDead(_4);
59+
+ StorageDead(_3);
60+
+ StorageDead(_2);
61+
+ _0 = const ();
62+
+ drop(_1) -> [return: bb1, unwind: bb3];
3863
}
3964
}
4065

‎tests/mir-opt/inline/inline_box_fn.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@
55
// EMIT_MIR inline_box_fn.call.Inline.diff
66
fn call(x: Box<dyn Fn(i32)>) {
77
// CHECK-LABEL: fn call(
8-
// CHECK-NOT: inlined
8+
// CHECK: (inlined <Box<dyn Fn(i32)> as Fn<(i32,)>>::call)
99
x(1);
1010
}

‎tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,15 @@
55
let mut _0: ();
66
let _1: ();
77
+ let mut _2: fn() {f};
8+
+ let mut _4: ();
89
+ scope 1 (inlined call::<fn() {f}>) {
910
+ debug f => _2;
1011
+ let _3: ();
12+
+ scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
13+
+ scope 3 (inlined f) {
14+
+ let _5: ();
15+
+ }
16+
+ }
1117
+ }
1218

1319
bb0: {
@@ -16,10 +22,15 @@
1622
+ StorageLive(_2);
1723
+ _2 = f;
1824
+ StorageLive(_3);
19-
+ _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, const ()) -> [return: bb1, unwind unreachable];
25+
+ StorageLive(_4);
26+
+ _4 = const ();
27+
+ StorageLive(_5);
28+
+ _5 = call::<fn() {f}>(f) -> [return: bb1, unwind unreachable];
2029
}
2130

2231
bb1: {
32+
+ StorageDead(_5);
33+
+ StorageDead(_4);
2334
+ StorageDead(_3);
2435
+ StorageDead(_2);
2536
StorageDead(_1);

0 commit comments

Comments
 (0)