Rollup merge of rust-lang#148240 - InvalidPathException:master, r=WaffleLapkin

rustc_codegen: fix musttail returns for cast/indirect ABIs

Fixes rust-lang#148239 rust-lang#144986

Explicit tail calls trigger `bug!` for any callee whose ABI returns via `PassMode::Cast`, and we forgot to to forward the hidden `sret` out-pointer when the ABI requested an indirect return. The former causes ICE, the latter produced malformed IR (wrong codegen) if the return value is large enough to need `sret`.

Updated the musttail helper to accept cast-mode returns, made it so that we pass the return pointer through the tail-call path.

Added two UI tests to demonstrate each case.

This is my first time contributing, please do check if I did it right.

r? theemathas

Author: matthiaskrgr

compiler/rustc_codegen_llvm/src/builder.rs

@@ -16,7 +16,6 @@ use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::*;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_hir::def_id::DefId;
-use rustc_middle::bug;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::ty::layout::{
     FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
@@ -1458,10 +1457,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
         match &fn_abi.ret.mode {
             PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(),
-            PassMode::Direct(_) | PassMode::Pair { .. } => self.ret(call),
-            mode @ PassMode::Cast { .. } => {
-                bug!("Encountered `PassMode::{mode:?}` during codegen")
-            }
+            PassMode::Direct(_) | PassMode::Pair { .. } | PassMode::Cast { .. } => self.ret(call),
         }
     }
 

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1063,7 +1063,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs);
                 target.map(|target| (return_dest, target))
             }
-            CallKind::Tail => None,
+            CallKind::Tail => {
+                if fn_abi.ret.is_indirect() {
+                    match self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs) {
+                        ReturnDest::Nothing => {}
+                        _ => bug!(
+                            "tail calls to functions with indirect returns cannot store into a destination"
+                        ),
+                    }
+                }
+                None
+            }
         };
 
         // Split the rust-call tupled arguments off.

tests/ui/explicit-tail-calls/become-cast-return.rs

@@ -0,0 +1,20 @@
+//@ run-pass
+//@ ignore-backends: gcc
+#![expect(incomplete_features)]
+#![feature(explicit_tail_calls)]
+
+#[inline(never)]
+fn leaf(_: &Box<u8>) -> [u8; 1] {
+    [1]
+}
+
+#[inline(never)]
+fn dispatch(param: &Box<u8>) -> [u8; 1] {
+    become leaf(param)
+}
+
+fn main() {
+    let data = Box::new(0);
+    let out = dispatch(&data);
+    assert_eq!(out, [1]);
+}

tests/ui/explicit-tail-calls/become-indirect-return.rs

@@ -0,0 +1,20 @@
+//@ run-pass
+//@ ignore-backends: gcc
+#![expect(incomplete_features)]
+#![feature(explicit_tail_calls)]
+
+#[inline(never)]
+fn op_dummy(_param: &Box<u8>) -> [u8; 24] {
+    [1; 24]
+}
+
+#[inline(never)]
+fn dispatch(param: &Box<u8>) -> [u8; 24] {
+    become op_dummy(param)
+}
+
+fn main() {
+    let param = Box::new(0);
+    let result = dispatch(&param);
+    assert_eq!(result, [1; 24]);
+}