Skip to content

Commit 5ded0f4

Browse files
uweiganduweigand
authored
Refactor call ABI implementation (#10722)
This refactors implementation of call ABI handling across architectures with the goal of bringing s390x in line with other platforms. The main idea is to - handle main call instruction selection and generation in ISLE (like s390x but unlike other platforms today) - handle argument setup mostly outside of ISLE (like other platforms but unlike s390x today) - handle return value processing as part of the call instructio (like all platforms today) All platforms now emit the main call instruction directly from ISLE, which e.g. handles selection of the correct ISA instruction depending on the call destination. This ISLE code calls out to helper routines to handle argument and return value processing. These helpers are mostly common code and provided by the Callee and/or Lower layers, with some platform-specific additions via ISLE Context routines. The old CallSite abstraction is no longer needed; most of the differences between call and return_call handling disappear. (There is still a common-code CallInfo vs. a platform-specifc ReturnCallInfo. At this point, it should be relatively straight- forward to make CallInfo platform-specific as well if desired, but this is not done here.) Some ISLE infrastructure for iterators / loops, which was only ever used by the s390x argument processing code, has been removed. s390x now closely matches all other platforms, with only a few special cases (slightly different tail-call ABI requires some differences in stack offset computations; we still need to handle vector lane swaps for cross-ABI calls), which should simplify future maintenance.
1 parent 1761bc3 commit 5ded0f4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1989
-2386
lines changed

‎cranelift/codegen/src/isa/aarch64/abi.rs‎

Lines changed: 25 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use crate::ir::types::*;
66
use crate::ir::MemFlags;
77
use crate::ir::{dynamic_to_fixed, ExternalName, LibCall, Signature};
88
use crate::isa;
9-
use crate::isa::aarch64::{inst::*, settings as aarch64_settings, AArch64Backend};
9+
use crate::isa::aarch64::{inst::*, settings as aarch64_settings};
1010
use crate::isa::unwind::UnwindInst;
1111
use crate::isa::winch;
1212
use crate::machinst::*;
@@ -25,9 +25,6 @@ use std::sync::OnceLock;
2525
/// Support for the AArch64 ABI from the callee side (within a function body).
2626
pub(crate) type AArch64Callee = Callee<AArch64MachineDeps>;
2727

28-
/// Support for the AArch64 ABI from the caller side (at a callsite).
29-
pub(crate) type AArch64CallSite = CallSite<AArch64MachineDeps>;
30-
3128
impl Into<AMode> for StackAMode {
3229
fn into(self) -> AMode {
3330
match self {
@@ -569,7 +566,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
569566
let setup_frame = frame_layout.setup_area_size > 0;
570567
let mut insts = SmallVec::new();
571568

572-
match select_api_key(isa_flags, call_conv, setup_frame) {
569+
match Self::select_api_key(isa_flags, call_conv, setup_frame) {
573570
Some(key) => {
574571
insts.push(Inst::Paci { key });
575572
if flags.unwind_info() {
@@ -675,7 +672,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
675672
) -> SmallInstVec<Inst> {
676673
let setup_frame = frame_layout.setup_area_size > 0;
677674

678-
match select_api_key(isa_flags, call_conv, setup_frame) {
675+
match Self::select_api_key(isa_flags, call_conv, setup_frame) {
679676
Some(key) => {
680677
smallvec![Inst::AuthenticatedRet {
681678
key,
@@ -1034,31 +1031,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
10341031
insts
10351032
}
10361033

1037-
fn gen_call(dest: &CallDest, tmp: Writable<Reg>, info: CallInfo<()>) -> SmallVec<[Inst; 2]> {
1038-
let mut insts = SmallVec::new();
1039-
match dest {
1040-
CallDest::ExtName(name, RelocDistance::Near) => {
1041-
let info = Box::new(info.map(|()| name.clone()));
1042-
insts.push(Inst::Call { info });
1043-
}
1044-
CallDest::ExtName(name, RelocDistance::Far) => {
1045-
insts.push(Inst::LoadExtName {
1046-
rd: tmp,
1047-
name: Box::new(name.clone()),
1048-
offset: 0,
1049-
});
1050-
let info = Box::new(info.map(|()| tmp.to_reg()));
1051-
insts.push(Inst::CallInd { info });
1052-
}
1053-
CallDest::Reg(reg) => {
1054-
let info = Box::new(info.map(|()| *reg));
1055-
insts.push(Inst::CallInd { info });
1056-
}
1057-
}
1058-
1059-
insts
1060-
}
1061-
10621034
fn gen_memcpy<F: FnMut(Type) -> Writable<Reg>>(
10631035
call_conv: isa::CallConv,
10641036
dst: Reg,
@@ -1257,89 +1229,30 @@ impl AArch64MachineDeps {
12571229
step: Imm12::maybe_from_u64(guard_size.into()).unwrap(),
12581230
});
12591231
}
1260-
}
12611232

1262-
fn select_api_key(
1263-
isa_flags: &aarch64_settings::Flags,
1264-
call_conv: isa::CallConv,
1265-
setup_frame: bool,
1266-
) -> Option<APIKey> {
1267-
if isa_flags.sign_return_address() && (setup_frame || isa_flags.sign_return_address_all()) {
1268-
// The `tail` calling convention uses a zero modifier rather than SP
1269-
// because tail calls may happen with a different stack pointer than
1270-
// when the function was entered, meaning that it won't be the same when
1271-
// the return address is decrypted.
1272-
Some(if isa_flags.sign_return_address_with_bkey() {
1273-
match call_conv {
1274-
isa::CallConv::Tail => APIKey::BZ,
1275-
_ => APIKey::BSP,
1276-
}
1233+
pub fn select_api_key(
1234+
isa_flags: &aarch64_settings::Flags,
1235+
call_conv: isa::CallConv,
1236+
setup_frame: bool,
1237+
) -> Option<APIKey> {
1238+
if isa_flags.sign_return_address() && (setup_frame || isa_flags.sign_return_address_all()) {
1239+
// The `tail` calling convention uses a zero modifier rather than SP
1240+
// because tail calls may happen with a different stack pointer than
1241+
// when the function was entered, meaning that it won't be the same when
1242+
// the return address is decrypted.
1243+
Some(if isa_flags.sign_return_address_with_bkey() {
1244+
match call_conv {
1245+
isa::CallConv::Tail => APIKey::BZ,
1246+
_ => APIKey::BSP,
1247+
}
1248+
} else {
1249+
match call_conv {
1250+
isa::CallConv::Tail => APIKey::AZ,
1251+
_ => APIKey::ASP,
1252+
}
1253+
})
12771254
} else {
1278-
match call_conv {
1279-
isa::CallConv::Tail => APIKey::AZ,
1280-
_ => APIKey::ASP,
1281-
}
1282-
})
1283-
} else {
1284-
None
1285-
}
1286-
}
1287-
1288-
impl AArch64CallSite {
1289-
pub fn emit_return_call(
1290-
mut self,
1291-
ctx: &mut Lower<Inst>,
1292-
args: isle::ValueSlice,
1293-
backend: &AArch64Backend,
1294-
) {
1295-
let new_stack_arg_size =
1296-
u32::try_from(self.sig(ctx.sigs()).sized_stack_arg_space()).unwrap();
1297-
1298-
ctx.abi_mut().accumulate_tail_args_size(new_stack_arg_size);
1299-
1300-
// Put all arguments in registers and stack slots (within that newly
1301-
// allocated stack space).
1302-
self.emit_args(ctx, args);
1303-
self.emit_stack_ret_arg_for_tail_call(ctx);
1304-
1305-
let dest = self.dest().clone();
1306-
let uses = self.take_uses();
1307-
let key = select_api_key(&backend.isa_flags, isa::CallConv::Tail, true);
1308-
1309-
match dest {
1310-
CallDest::ExtName(callee, RelocDistance::Near) => {
1311-
let info = Box::new(ReturnCallInfo {
1312-
dest: callee,
1313-
uses,
1314-
key,
1315-
new_stack_arg_size,
1316-
});
1317-
ctx.emit(Inst::ReturnCall { info });
1318-
}
1319-
CallDest::ExtName(name, RelocDistance::Far) => {
1320-
let callee = ctx.alloc_tmp(types::I64).only_reg().unwrap();
1321-
ctx.emit(Inst::LoadExtName {
1322-
rd: callee,
1323-
name: Box::new(name),
1324-
offset: 0,
1325-
});
1326-
let info = Box::new(ReturnCallInfo {
1327-
dest: callee.to_reg(),
1328-
uses,
1329-
key,
1330-
new_stack_arg_size,
1331-
});
1332-
ctx.emit(Inst::ReturnCallInd { info });
1333-
}
1334-
CallDest::Reg(callee) => {
1335-
let info = Box::new(ReturnCallInfo {
1336-
dest: callee,
1337-
uses,
1338-
key,
1339-
new_stack_arg_size,
1340-
});
1341-
ctx.emit(Inst::ReturnCallInd { info });
1342-
}
1255+
None
13431256
}
13441257
}
13451258
}

‎cranelift/codegen/src/isa/aarch64/inst.isle‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4408,17 +4408,37 @@
44084408

44094409
;;;; Helpers for Emitting Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
44104410

4411-
(decl gen_call (SigRef ExternalName RelocDistance ValueSlice) InstOutput)
4412-
(extern constructor gen_call gen_call)
4411+
(decl gen_call_info (Sig ExternalName CallArgList CallRetList OptionTryCallInfo) BoxCallInfo)
4412+
(extern constructor gen_call_info gen_call_info)
44134413

4414-
(decl gen_call_indirect (SigRef Value ValueSlice) InstOutput)
4415-
(extern constructor gen_call_indirect gen_call_indirect)
4414+
(decl gen_call_ind_info (Sig Reg CallArgList CallRetList OptionTryCallInfo) BoxCallIndInfo)
4415+
(extern constructor gen_call_ind_info gen_call_ind_info)
44164416

4417-
(decl gen_try_call (SigRef ExternalName RelocDistance ExceptionTable ValueSlice MachLabelSlice) Unit)
4418-
(extern constructor gen_try_call gen_try_call)
4417+
(decl gen_return_call_info (Sig ExternalName CallArgList) BoxReturnCallInfo)
4418+
(extern constructor gen_return_call_info gen_return_call_info)
44194419

4420-
(decl gen_try_call_indirect (SigRef Value ExceptionTable ValueSlice MachLabelSlice) Unit)
4421-
(extern constructor gen_try_call_indirect gen_try_call_indirect)
4420+
(decl gen_return_call_ind_info (Sig Reg CallArgList) BoxReturnCallIndInfo)
4421+
(extern constructor gen_return_call_ind_info gen_return_call_ind_info)
4422+
4423+
;; Helper for creating `MInst.Call` instructions.
4424+
(decl call_impl (BoxCallInfo) SideEffectNoResult)
4425+
(rule (call_impl info)
4426+
(SideEffectNoResult.Inst (MInst.Call info)))
4427+
4428+
;; Helper for creating `MInst.CallInd` instructions.
4429+
(decl call_ind_impl (BoxCallIndInfo) SideEffectNoResult)
4430+
(rule (call_ind_impl info)
4431+
(SideEffectNoResult.Inst (MInst.CallInd info)))
4432+
4433+
;; Helper for creating `MInst.ReturnCall` instructions.
4434+
(decl return_call_impl (BoxReturnCallInfo) SideEffectNoResult)
4435+
(rule (return_call_impl info)
4436+
(SideEffectNoResult.Inst (MInst.ReturnCall info)))
4437+
4438+
;; Helper for creating `MInst.ReturnCallInd` instructions.
4439+
(decl return_call_ind_impl (BoxReturnCallIndInfo) SideEffectNoResult)
4440+
(rule (return_call_ind_impl info)
4441+
(SideEffectNoResult.Inst (MInst.ReturnCallInd info)))
44224442

44234443
;; Helpers for pinned register manipulation.
44244444

‎cranelift/codegen/src/isa/aarch64/lower.isle‎

Lines changed: 86 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2503,20 +2503,71 @@
25032503
(rule (lower (get_return_address))
25042504
(aarch64_link))
25052505

2506-
;;;; Rules for calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2507-
2508-
(rule (lower (call (func_ref_data sig_ref extname dist) inputs))
2509-
(gen_call sig_ref extname dist inputs))
2510-
2511-
(rule (lower (call_indirect sig_ref val inputs))
2512-
(gen_call_indirect sig_ref val inputs))
2513-
2514-
(rule (lower_branch (try_call (func_ref_data sig_ref extname dist) inputs et) targets)
2515-
(gen_try_call sig_ref extname dist et inputs targets))
2516-
2517-
(rule (lower_branch (try_call_indirect val inputs et) targets)
2506+
;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2507+
2508+
;; Direct call to an in-range function.
2509+
(rule 1 (lower (call (func_ref_data sig_ref name (reloc_distance_near)) args))
2510+
(let ((output ValueRegsVec (gen_call_output sig_ref))
2511+
(abi Sig (abi_sig sig_ref))
2512+
(uses CallArgList (gen_call_args abi args))
2513+
(defs CallRetList (gen_call_rets abi output))
2514+
(info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))
2515+
(_ Unit (emit_side_effect (call_impl info))))
2516+
output))
2517+
2518+
;; Direct call to an out-of-range function (implicitly via pointer).
2519+
(rule (lower (call (func_ref_data sig_ref name _) args))
2520+
(let ((output ValueRegsVec (gen_call_output sig_ref))
2521+
(abi Sig (abi_sig sig_ref))
2522+
(uses CallArgList (gen_call_args abi args))
2523+
(defs CallRetList (gen_call_rets abi output))
2524+
(target Reg (load_ext_name name 0))
2525+
(info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none)))
2526+
(_ Unit (emit_side_effect (call_ind_impl info))))
2527+
output))
2528+
2529+
;; Indirect call.
2530+
(rule (lower (call_indirect sig_ref ptr args))
2531+
(let ((output ValueRegsVec (gen_call_output sig_ref))
2532+
(abi Sig (abi_sig sig_ref))
2533+
(target Reg (put_in_reg ptr))
2534+
(uses CallArgList (gen_call_args abi args))
2535+
(defs CallRetList (gen_call_rets abi output))
2536+
(info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none)))
2537+
(_ Unit (emit_side_effect (call_ind_impl info))))
2538+
output))
2539+
2540+
;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2541+
2542+
;; Direct call to an in-range function.
2543+
(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (reloc_distance_near)) args et) targets)
2544+
(let ((abi Sig (abi_sig sig_ref))
2545+
(trycall OptionTryCallInfo (try_call_info et targets))
2546+
(uses CallArgList (gen_call_args abi args))
2547+
(defs CallRetList (gen_try_call_rets abi))
2548+
(info BoxCallInfo (gen_call_info abi name uses defs trycall)))
2549+
(emit_side_effect (call_impl info))))
2550+
2551+
;; Direct call to an out-of-range function (implicitly via pointer).
2552+
(rule (lower_branch (try_call (func_ref_data sig_ref name _) args et) targets)
2553+
(let ((abi Sig (abi_sig sig_ref))
2554+
(trycall OptionTryCallInfo (try_call_info et targets))
2555+
(uses CallArgList (gen_call_args abi args))
2556+
(defs CallRetList (gen_try_call_rets abi))
2557+
(target Reg (load_ext_name name 0))
2558+
(info BoxCallIndInfo (gen_call_ind_info abi target uses defs trycall)))
2559+
(emit_side_effect (call_ind_impl info))))
2560+
2561+
;; Indirect call.
2562+
(rule (lower_branch (try_call_indirect ptr args et) targets)
25182563
(if-let (exception_sig sig_ref) et)
2519-
(gen_try_call_indirect sig_ref val et inputs targets))
2564+
(let ((abi Sig (abi_sig sig_ref))
2565+
(trycall OptionTryCallInfo (try_call_info et targets))
2566+
(target Reg (put_in_reg ptr))
2567+
(uses CallArgList (gen_call_args abi args))
2568+
(defs CallRetList (gen_try_call_rets abi))
2569+
(info BoxCallIndInfo (gen_call_ind_info abi target uses defs trycall)))
2570+
(emit_side_effect (call_ind_impl info))))
25202571

25212572
;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
25222573

@@ -2526,11 +2577,28 @@
25262577

25272578
;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;
25282579

2529-
(rule (lower (return_call (func_ref_data sig_ref extname dist) args))
2530-
(gen_return_call sig_ref extname dist args))
2531-
2532-
(rule (lower (return_call_indirect sig_ref callee args))
2533-
(gen_return_call_indirect sig_ref callee args))
2580+
;; Direct call to an in-range function.
2581+
(rule 1 (lower (return_call (func_ref_data sig_ref name (reloc_distance_near)) args))
2582+
(let ((abi Sig (abi_sig sig_ref))
2583+
(uses CallArgList (gen_return_call_args abi args))
2584+
(info BoxReturnCallInfo (gen_return_call_info abi name uses)))
2585+
(side_effect (return_call_impl info))))
2586+
2587+
;; Direct call to an out-of-range function (implicitly via pointer).
2588+
(rule (lower (return_call (func_ref_data sig_ref name _) args))
2589+
(let ((abi Sig (abi_sig sig_ref))
2590+
(uses CallArgList (gen_return_call_args abi args))
2591+
(target Reg (load_ext_name name 0))
2592+
(info BoxReturnCallIndInfo (gen_return_call_ind_info abi target uses)))
2593+
(side_effect (return_call_ind_impl info))))
2594+
2595+
;; Indirect call.
2596+
(rule (lower (return_call_indirect sig_ref ptr args))
2597+
(let ((abi Sig (abi_sig sig_ref))
2598+
(target Reg (put_in_reg ptr))
2599+
(uses CallArgList (gen_return_call_args abi args))
2600+
(info BoxReturnCallIndInfo (gen_return_call_ind_info abi target uses)))
2601+
(side_effect (return_call_ind_impl info))))
25342602

25352603
;;;; Rules for loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
25362604

0 commit comments

Comments
 (0)