[s390x] Fix SP unwind rule for the tail-call ABI

cranelift/codegen/src/isa/s390x/abi.rs

@@ -738,14 +738,29 @@ impl ABIMachineSpec for S390xMachineDeps {
         // Use STMG to save clobbered GPRs into save area.
         // Note that we always save SP (%r15) here if anything is saved.
         if let Some((first_clobbered_gpr, _)) = get_clobbered_gprs(frame_layout) {
+            let mut last_clobbered_gpr = 15;
             let offset = 8 * first_clobbered_gpr as i64 + incoming_tail_args_size as i64;
             insts.push(Inst::StoreMultiple64 {
                 rt: gpr(first_clobbered_gpr),
-                rt2: gpr(15),
+                rt2: gpr(last_clobbered_gpr),
                 mem: MemArg::reg_plus_off(stack_reg(), offset, MemFlags::trusted()),
             });
             if flags.unwind_info() {
-                for i in first_clobbered_gpr..16 {
+                // Normally, we instruct the unwinder to restore the stack pointer
+                // from its slot in the save area.  However, if we have incoming
+                // tail-call arguments, the value saved in that slot is incorrect.
+                // In that case, we instead instruct the unwinder to compute the
+                // unwound SP relative to the current CFA, as CFA == SP + 160.
+                if incoming_tail_args_size != 0 {
+                    insts.push(Inst::Unwind {
+                        inst: UnwindInst::RegStackOffset {
+                            clobber_offset: frame_layout.clobber_size,
+                            reg: gpr(last_clobbered_gpr).to_real_reg().unwrap(),
+                        },
+                    });
+                    last_clobbered_gpr = last_clobbered_gpr - 1;
+                }
+                for i in first_clobbered_gpr..(last_clobbered_gpr + 1) {
                     insts.push(Inst::Unwind {
                         inst: UnwindInst::SaveReg {
                             clobber_offset: frame_layout.clobber_size + (i * 8) as u32,

cranelift/codegen/src/isa/unwind.rs

@@ -196,6 +196,15 @@ pub enum UnwindInst {
         /// The saved register.
         reg: RealReg,
     },
+    /// Computes the value of the given register in the caller as stack offset.
+    /// Typically used to unwind the stack pointer if the default rule does not apply.
+    /// The `clobber_offset` is computed the same way as for the `SaveReg` rule.
+    RegStackOffset {
+        /// The offset from the start of the clobber area to this register's value.
+        clobber_offset: u32,
+        /// The register whose value is to be set.
+        reg: RealReg,
+    },
     /// Defines if the aarch64-specific pointer authentication available for ARM v8.3+ devices
     /// is enabled for certain pointers or not.
     Aarch64SetPointerAuth {

cranelift/codegen/src/isa/unwind/systemv.rs

@@ -247,6 +247,19 @@ pub(crate) fn create_unwind_info_from_insts<MR: RegisterMapper<Reg>>(
                 let off = (clobber_offset as i32) - (clobber_offset_to_cfa as i32);
                 instructions.push((instruction_offset, CallFrameInstruction::Offset(reg, off)));
             }
+            &UnwindInst::RegStackOffset {
+                clobber_offset,
+                reg,
+            } => {
+                let reg = mr
+                    .map(reg.into())
+                    .map_err(|e| CodegenError::RegisterMappingError(e))?;
+                let off = (clobber_offset as i32) - (clobber_offset_to_cfa as i32);
+                instructions.push((
+                    instruction_offset,
+                    CallFrameInstruction::ValOffset(reg, off),
+                ));
+            }
             &UnwindInst::Aarch64SetPointerAuth { return_addresses } => {
                 instructions.push((
                     instruction_offset,

cranelift/codegen/src/isa/unwind/winarm64.rs

@@ -301,6 +301,9 @@ pub(crate) fn create_unwind_info_from_insts(
                     }
                 }
             }
+            &UnwindInst::RegStackOffset { .. } => {
+                unreachable!("only supported with DWARF");
+            }
             &UnwindInst::Aarch64SetPointerAuth { return_addresses } => {
                 assert!(
                     return_addresses,

cranelift/codegen/src/isa/unwind/winx64.rs

@@ -285,6 +285,9 @@ pub(crate) fn create_unwind_info_from_insts<MR: RegisterMapper<crate::machinst::
                     });
                 }
             },
+            &UnwindInst::RegStackOffset { .. } => {
+                unreachable!("only supported with DWARF");
+            }
             &UnwindInst::Aarch64SetPointerAuth { .. } => {
                 unreachable!("no aarch64 on x64");
             }