Cranelift: update to latest regalloc2: (bytecodealliance#4324)

- Handle call instructions' clobbers with the clobbers API, using RA2's
  clobbers bitmask (bytecodealliance/regalloc2#58) rather than clobbers
  list;

- Pull in changes from bytecodealliance/regalloc2#59 for much more sane
  edge-case behavior w.r.t. liverange splitting.

cranelift/codegen/Cargo.toml

@@ -23,7 +23,7 @@ serde = { version = "1.0.94", features = ["derive"], optional = true }
 bincode = { version = "1.2.1", optional = true }
 gimli = { version = "0.26.0", default-features = false, features = ["write"], optional = true }
 smallvec = { version = "1.6.1" }
-regalloc2 = { version = "0.2.3", features = ["checker"] }
+regalloc2 = { version = "0.3.0", features = ["checker"] }
 souper-ir = { version = "2.1.0", optional = true }
 # It is a goal of the cranelift-codegen crate to have minimal external dependencies.
 # Please don't add any unless they are essential to the task of creating binary

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -14,7 +14,7 @@ use crate::settings;
 use crate::{CodegenError, CodegenResult};
 use alloc::boxed::Box;
 use alloc::vec::Vec;
-use regalloc2::VReg;
+use regalloc2::{PRegSet, VReg};
 use smallvec::{smallvec, SmallVec};
 
 // We use a generic implementation that factors out AArch64 and x64 ABI commonalities, because
@@ -1062,8 +1062,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
 
  fn gen_call(
  dest: &CallDest,
- uses: Vec<Reg>,
- defs: Vec<Writable<Reg>>,
+ uses: SmallVec<[Reg; 8]>,
+ defs: SmallVec<[Writable<Reg>; 8]>,
+ clobbers: PRegSet,
  opcode: ir::Opcode,
  tmp: Writable<Reg>,
  callee_conv: isa::CallConv,
@@ -1076,6 +1077,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
  dest: name.clone(),
  uses,
  defs,
+ clobbers,
  opcode,
  caller_callconv: caller_conv,
  callee_callconv: callee_conv,
@@ -1092,6 +1094,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
  rn: tmp.to_reg(),
  uses,
  defs,
+ clobbers,
  opcode,
  caller_callconv: caller_conv,
  callee_callconv: callee_conv,
@@ -1103,6 +1106,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
  rn: *reg,
  uses,
  defs,
+ clobbers,
  opcode,
  caller_callconv: caller_conv,
  callee_callconv: callee_conv,
@@ -1131,8 +1135,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
  insts.push(Inst::Call {
  info: Box::new(CallInfo {
  dest: ExternalName::LibCall(LibCall::Memcpy),
- uses: vec![arg0.to_reg(), arg1.to_reg(), arg2.to_reg()],
- defs: Self::get_regs_clobbered_by_call(call_conv),
+ uses: smallvec![arg0.to_reg(), arg1.to_reg(), arg2.to_reg()],
+ defs: smallvec![],
+ clobbers: Self::get_regs_clobbered_by_call(call_conv),
  opcode: Opcode::Call,
  caller_callconv: call_conv,
  callee_callconv: call_conv,
@@ -1159,21 +1164,19 @@ impl ABIMachineSpec for AArch64MachineDeps {
  s.nominal_sp_to_fp
  }
 
- fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> Vec<Writable<Reg>> {
- let mut caller_saved = Vec::new();
- for i in 0..29 {
- let x = writable_xreg(i);
- if is_reg_clobbered_by_call(call_conv_of_callee, x.to_reg().to_real_reg().unwrap()) {
- caller_saved.push(x);
- }
- }
- for i in 0..32 {
- let v = writable_vreg(i);
- if is_reg_clobbered_by_call(call_conv_of_callee, v.to_reg().to_real_reg().unwrap()) {
- caller_saved.push(v);
+ fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> PRegSet {
+ let mut clobbers = DEFAULT_AAPCS_CLOBBERS;
+
+ if call_conv_of_callee.extends_baldrdash() {
+ // Every X-register except for x16, x17, x18 is
+ // caller-saved (clobbered by a call).
+ for i in 19..=28 {
+ clobbers.add(xreg_preg(i));
  }
+ clobbers.add(vreg_preg(31));
  }
- caller_saved
+
+ clobbers
  }
 
  fn get_ext_mode(
@@ -1290,47 +1293,74 @@ fn get_regs_restored_in_epilogue(
  (int_saves, vec_saves)
 }
 
-fn is_reg_clobbered_by_call(call_conv_of_callee: isa::CallConv, r: RealReg) -> bool {
- if call_conv_of_callee.extends_baldrdash() {
- match r.class() {
- RegClass::Int => {
- let enc = r.hw_enc() & 31;
- if !BALDRDASH_JIT_CALLEE_SAVED_GPR[enc as usize] {
- return true;
- }
- // Otherwise, fall through to preserve native's ABI caller-saved.
- }
- RegClass::Float => {
- let enc = r.hw_enc() & 31;
- if !BALDRDASH_JIT_CALLEE_SAVED_FPU[enc as usize] {
- return true;
- }
- // Otherwise, fall through to preserve native's ABI caller-saved.
- }
- };
- }
-
- match r.class() {
- RegClass::Int => {
- // x0 - x17 inclusive are caller-saves.
- r.hw_enc() <= 17
- }
- RegClass::Float => {
- // v0 - v7 inclusive and v16 - v31 inclusive are caller-saves. The
- // upper 64 bits of v8 - v15 inclusive are also caller-saves.
- // However, because we cannot currently represent partial registers
- // to regalloc.rs, we indicate here that every vector register is
- // caller-save. Because this function is used at *callsites*,
- // approximating in this direction (save more than necessary) is
- // conservative and thus safe.
- //
- // Note that we set the 'not included in clobber set' flag in the
- // regalloc.rs API when a call instruction's callee has the same ABI
- // as the caller (the current function body); this is safe (anything
- // clobbered by callee can be clobbered by caller as well) and
- // avoids unnecessary saves of v8-v15 in the prologue even though we
- // include them as defs here.
- true
- }
- }
+const fn default_aapcs_clobbers() -> PRegSet {
+ PRegSet::empty()
+ // x0 - x17 inclusive are caller-saves.
+ .with(xreg_preg(0))
+ .with(xreg_preg(1))
+ .with(xreg_preg(2))
+ .with(xreg_preg(3))
+ .with(xreg_preg(4))
+ .with(xreg_preg(5))
+ .with(xreg_preg(6))
+ .with(xreg_preg(7))
+ .with(xreg_preg(8))
+ .with(xreg_preg(9))
+ .with(xreg_preg(10))
+ .with(xreg_preg(11))
+ .with(xreg_preg(12))
+ .with(xreg_preg(13))
+ .with(xreg_preg(14))
+ .with(xreg_preg(15))
+ .with(xreg_preg(16))
+ .with(xreg_preg(17))
+ // v0 - v7 inclusive and v16 - v31 inclusive are
+ // caller-saves. The upper 64 bits of v8 - v15 inclusive are
+ // also caller-saves. However, because we cannot currently
+ // represent partial registers to regalloc2, we indicate here
+ // that every vector register is caller-save. Because this
+ // function is used at *callsites*, approximating in this
+ // direction (save more than necessary) is conservative and
+ // thus safe.
+ //
+ // Note that we exclude clobbers from a call instruction when
+ // a call instruction's callee has the same ABI as the caller
+ // (the current function body); this is safe (anything
+ // clobbered by callee can be clobbered by caller as well) and
+ // avoids unnecessary saves of v8-v15 in the prologue even
+ // though we include them as defs here.
+ .with(vreg_preg(0))
+ .with(vreg_preg(1))
+ .with(vreg_preg(2))
+ .with(vreg_preg(3))
+ .with(vreg_preg(4))
+ .with(vreg_preg(5))
+ .with(vreg_preg(6))
+ .with(vreg_preg(7))
+ .with(vreg_preg(8))
+ .with(vreg_preg(9))
+ .with(vreg_preg(10))
+ .with(vreg_preg(11))
+ .with(vreg_preg(12))
+ .with(vreg_preg(13))
+ .with(vreg_preg(14))
+ .with(vreg_preg(15))
+ .with(vreg_preg(16))
+ .with(vreg_preg(17))
+ .with(vreg_preg(18))
+ .with(vreg_preg(19))
+ .with(vreg_preg(20))
+ .with(vreg_preg(21))
+ .with(vreg_preg(22))
+ .with(vreg_preg(23))
+ .with(vreg_preg(24))
+ .with(vreg_preg(25))
+ .with(vreg_preg(26))
+ .with(vreg_preg(27))
+ .with(vreg_preg(28))
+ .with(vreg_preg(29))
+ .with(vreg_preg(30))
+ .with(vreg_preg(31))
 }
+
+const DEFAULT_AAPCS_CLOBBERS: PRegSet = default_aapcs_clobbers();

cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs

@@ -5286,8 +5286,9 @@ fn test_aarch64_binemit() {
  Inst::Call {
  info: Box::new(CallInfo {
  dest: ExternalName::testcase("test0"),
- uses: Vec::new(),
- defs: Vec::new(),
+ uses: smallvec![],
+ defs: smallvec![],
+ clobbers: PRegSet::empty(),
  opcode: Opcode::Call,
  caller_callconv: CallConv::SystemV,
  callee_callconv: CallConv::SystemV,
@@ -5301,8 +5302,9 @@ fn test_aarch64_binemit() {
  Inst::CallInd {
  info: Box::new(CallIndInfo {
  rn: xreg(10),
- uses: Vec::new(),
- defs: Vec::new(),
+ uses: smallvec![],
+ defs: smallvec![],
+ clobbers: PRegSet::empty(),
  opcode: Opcode::CallIndirect,
  caller_callconv: CallConv::SystemV,
  callee_callconv: CallConv::SystemV,

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -13,7 +13,7 @@ use crate::machinst::{PrettyPrint, Reg, RegClass, Writable};
 
 use alloc::vec::Vec;
 use core::convert::TryFrom;
-use regalloc2::VReg;
+use regalloc2::{PRegSet, VReg};
 use smallvec::{smallvec, SmallVec};
 use std::string::{String, ToString};
 
@@ -70,8 +70,9 @@ impl BitOp {
 #[derive(Clone, Debug)]
 pub struct CallInfo {
  pub dest: ExternalName,
- pub uses: Vec<Reg>,
- pub defs: Vec<Writable<Reg>>,
+ pub uses: SmallVec<[Reg; 8]>,
+ pub defs: SmallVec<[Writable<Reg>; 8]>,
+ pub clobbers: PRegSet,
  pub opcode: Opcode,
  pub caller_callconv: CallConv,
  pub callee_callconv: CallConv,
@@ -82,8 +83,9 @@ pub struct CallInfo {
 #[derive(Clone, Debug)]
 pub struct CallIndInfo {
  pub rn: Reg,
- pub uses: Vec<Reg>,
- pub defs: Vec<Writable<Reg>>,
+ pub uses: SmallVec<[Reg; 8]>,
+ pub defs: SmallVec<[Writable<Reg>; 8]>,
+ pub clobbers: PRegSet,
  pub opcode: Opcode,
  pub caller_callconv: CallConv,
  pub callee_callconv: CallConv,
@@ -983,11 +985,13 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
  &Inst::Call { ref info, .. } => {
  collector.reg_uses(&info.uses[..]);
  collector.reg_defs(&info.defs[..]);
+ collector.reg_clobbers(info.clobbers);
  }
  &Inst::CallInd { ref info, .. } => {
  collector.reg_use(info.rn);
  collector.reg_uses(&info.uses[..]);
  collector.reg_defs(&info.defs[..]);
+ collector.reg_clobbers(info.clobbers);
  }
  &Inst::CondBr { ref kind, .. } => match kind {
  CondBrKind::Zero(rt) | CondBrKind::NotZero(rt) => {
@@ -1028,9 +1032,9 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
  &Inst::VirtualSPOffsetAdj { .. } => {}
 
  &Inst::ElfTlsGetAddr { .. } => {
- for reg in AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::SystemV) {
- collector.reg_def(reg);
- }
+ collector.reg_clobbers(AArch64MachineDeps::get_regs_clobbered_by_call(
+ CallConv::SystemV,
+ ));
  }
  &Inst::Unwind { .. } => {}
  &Inst::EmitIsland { .. } => {}

cranelift/codegen/src/isa/aarch64/inst/regs.rs

@@ -24,9 +24,13 @@ pub const PINNED_REG: u8 = 21;
 /// Get a reference to an X-register (integer register). Do not use
 /// this for xsp / xzr; we have two special registers for those.
 pub fn xreg(num: u8) -> Reg {
+ Reg::from(xreg_preg(num))
+}
+
+/// Get the given X-register as a PReg.
+pub(crate) const fn xreg_preg(num: u8) -> PReg {
  assert!(num < 31);
- let preg = PReg::new(num as usize, RegClass::Int);
- Reg::from(VReg::new(preg.index(), RegClass::Int))
+ PReg::new(num as usize, RegClass::Int)
 }
 
 /// Get a writable reference to an X-register.
@@ -36,9 +40,13 @@ pub fn writable_xreg(num: u8) -> Writable<Reg> {
 
 /// Get a reference to a V-register (vector/FP register).
 pub fn vreg(num: u8) -> Reg {
+ Reg::from(vreg_preg(num))
+}
+
+/// Get the given V-register as a PReg.
+pub(crate) const fn vreg_preg(num: u8) -> PReg {
  assert!(num < 32);
- let preg = PReg::new(num as usize, RegClass::Float);
- Reg::from(VReg::new(preg.index(), RegClass::Float))
+ PReg::new(num as usize, RegClass::Float)
 }
 
 /// Get a writable reference to a V-register.