From fde77a6674b73bb1b2ade7b09392bd5c157a4aa8 Mon Sep 17 00:00:00 2001 From: Anton Kirilov Date: Wed, 3 Aug 2022 21:37:38 +0000 Subject: [PATCH] Cranelift AArch64: Migrate AMode to ISLE Copyright (c) 2022, Arm Limited. --- cranelift/codegen/src/isa/aarch64/abi.rs | 44 ++- cranelift/codegen/src/isa/aarch64/inst.isle | 150 ++++++++- .../codegen/src/isa/aarch64/inst/args.rs | 220 ++++++------- .../codegen/src/isa/aarch64/inst/emit.rs | 171 +++++----- .../src/isa/aarch64/inst/emit_tests.rs | 309 ++++++++++++++---- cranelift/codegen/src/isa/aarch64/inst/mod.rs | 44 ++- cranelift/codegen/src/isa/aarch64/lower.isle | 9 +- cranelift/codegen/src/isa/aarch64/lower.rs | 56 +++- .../codegen/src/isa/aarch64/lower/isle.rs | 40 ++- 9 files changed, 705 insertions(+), 338 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 805287460337..3b8d0cc11baf 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -34,9 +34,9 @@ static STACK_ARG_RET_SIZE_LIMIT: u64 = 128 * 1024 * 1024; impl Into for StackAMode { fn into(self) -> AMode { match self { - StackAMode::FPOffset(off, ty) => AMode::FPOffset(off, ty), - StackAMode::NominalSPOffset(off, ty) => AMode::NominalSPOffset(off, ty), - StackAMode::SPOffset(off, ty) => AMode::SPOffset(off, ty), + StackAMode::FPOffset(off, ty) => AMode::FPOffset { off, ty }, + StackAMode::NominalSPOffset(off, ty) => AMode::NominalSPOffset { off, ty }, + StackAMode::SPOffset(off, ty) => AMode::SPOffset { off, ty }, } } } @@ -462,12 +462,20 @@ impl ABIMachineSpec for AArch64MachineDeps { } fn gen_load_base_offset(into_reg: Writable, base: Reg, offset: i32, ty: Type) -> Inst { - let mem = AMode::RegOffset(base, offset as i64, ty); + let mem = AMode::RegOffset { + rn: base, + off: offset as i64, + ty, + }; Inst::gen_load(into_reg, mem, ty, MemFlags::trusted()) } fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Inst { - let mem = AMode::RegOffset(base, offset as i64, ty); + let mem = AMode::RegOffset { + rn: base, + off: offset as i64, + ty, + }; Inst::gen_store(mem, from_reg, ty, MemFlags::trusted()) } @@ -676,10 +684,10 @@ impl ABIMachineSpec for AArch64MachineDeps { // str rd, [sp, #-16]! insts.push(Inst::Store64 { rd, - mem: AMode::PreIndexed( - writable_stack_reg(), - SImm9::maybe_from_i64(-clobber_offset_change).unwrap(), - ), + mem: AMode::PreIndexed { + rn: writable_stack_reg(), + simm9: SImm9::maybe_from_i64(-clobber_offset_change).unwrap(), + }, flags: MemFlags::trusted(), }); @@ -734,10 +742,10 @@ impl ABIMachineSpec for AArch64MachineDeps { let store_vec_reg = |rd| Inst::FpuStore64 { rd, - mem: AMode::PreIndexed( - writable_stack_reg(), - SImm9::maybe_from_i64(-clobber_offset_change).unwrap(), - ), + mem: AMode::PreIndexed { + rn: writable_stack_reg(), + simm9: SImm9::maybe_from_i64(-clobber_offset_change).unwrap(), + }, flags: MemFlags::trusted(), }; let iter = clobbered_vec.chunks_exact(2); @@ -831,7 +839,10 @@ impl ABIMachineSpec for AArch64MachineDeps { let load_vec_reg = |rd| Inst::FpuLoad64 { rd, - mem: AMode::PostIndexed(writable_stack_reg(), SImm9::maybe_from_i64(16).unwrap()), + mem: AMode::PostIndexed { + rn: writable_stack_reg(), + simm9: SImm9::maybe_from_i64(16).unwrap(), + }, flags: MemFlags::trusted(), }; let load_vec_reg_pair = |rt, rt2| Inst::FpuLoadP64 { @@ -893,7 +904,10 @@ impl ABIMachineSpec for AArch64MachineDeps { // ldr rd, [sp], #16 insts.push(Inst::ULoad64 { rd, - mem: AMode::PostIndexed(writable_stack_reg(), SImm9::maybe_from_i64(16).unwrap()), + mem: AMode::PostIndexed { + rn: writable_stack_reg(), + simm9: SImm9::maybe_from_i64(16).unwrap(), + }, flags: MemFlags::trusted(), }); } diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index a49f8872d12c..731a4c46cb0f 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -931,7 +931,108 @@ (Cls) )) -(type AMode extern (enum)) +(type MemLabel extern (enum)) +(type SImm9 extern (enum)) +(type UImm12Scaled extern (enum)) + +;; An addressing mode specified for a load/store operation. +(type AMode + (enum + ;; + ;; Real ARM64 addressing modes: + ;; + ;; "post-indexed" mode as per AArch64 docs: postincrement reg after + ;; address computation. + (PostIndexed + (rn WritableReg) + (simm9 SImm9)) + + ;; "pre-indexed" mode as per AArch64 docs: preincrement reg before + ;; address computation. + (PreIndexed + (rn WritableReg) + (simm9 SImm9)) + + ;; N.B.: RegReg, RegScaled, and RegScaledExtended all correspond to + ;; what the ISA calls the "register offset" addressing mode. We split + ;; out several options here for more ergonomic codegen. + ;; + ;; Register plus register offset. + (RegReg + (rn Reg) + (rm Reg)) + + ;; Register plus register offset, scaled by type's size. + (RegScaled + (rn Reg) + (rm Reg) + (ty Type)) + + ;; Register plus register offset, scaled by type's size, with index + ;; sign- or zero-extended first. + (RegScaledExtended + (rn Reg) + (rm Reg) + (ty Type) + (extendop ExtendOp)) + + ;; Register plus register offset, with index sign- or zero-extended + ;; first. + (RegExtended + (rn Reg) + (rm Reg) + (extendop ExtendOp)) + + ;; Unscaled signed 9-bit immediate offset from reg. + (Unscaled + (rn Reg) + (simm9 SImm9)) + + ;; Scaled (by size of a type) unsigned 12-bit immediate offset from reg. + (UnsignedOffset + (rn Reg) + (uimm12 UImm12Scaled)) + + ;; virtual addressing modes that are lowered at emission time: + ;; + ;; Reference to a "label": e.g., a symbol. + (Label + (label MemLabel)) + + ;; Arbitrary offset from a register. Converted to generation of large + ;; offsets with multiple instructions as necessary during code emission. + (RegOffset + (rn Reg) + (off i64) + (ty Type)) + + ;; Offset from the stack pointer. + (SPOffset + (off i64) + (ty Type)) + + ;; Offset from the frame pointer. + (FPOffset + (off i64) + (ty Type)) + + ;; Offset from the "nominal stack pointer", which is where the real SP is + ;; just after stack and spill slots are allocated in the function prologue. + ;; At emission time, this is converted to `SPOffset` with a fixup added to + ;; the offset constant. The fixup is a running value that is tracked as + ;; emission iterates through instructions in linear order, and can be + ;; adjusted up and down with [Inst::VirtualSPOffsetAdj]. + ;; + ;; The standard ABI is in charge of handling this (by emitting the + ;; adjustment meta-instructions). It maintains the invariant that "nominal + ;; SP" is where the actual SP is after the function prologue and before + ;; clobber pushes. See the diagram in the documentation for + ;; [crate::isa::aarch64::abi](the ABI module) for more details. + (NominalSPOffset + (off i64) + (ty Type)) +)) + (type PairAMode extern (enum)) (type FPUOpRI extern (enum)) @@ -1470,6 +1571,12 @@ (decl valid_atomic_transaction (Type) Type) (extern extractor valid_atomic_transaction valid_atomic_transaction) +(decl pure is_zero_simm9 (SImm9) Unit) +(extern constructor is_zero_simm9 is_zero_simm9) + +(decl pure is_zero_uimm12 (UImm12Scaled) Unit) +(extern constructor is_zero_uimm12 is_zero_uimm12) + ;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`. (decl imm12_from_value (Imm12) Value) (extractor @@ -1502,25 +1609,21 @@ (decl cond_br_cond (Cond) CondBrKind) (extern constructor cond_br_cond cond_br_cond) -;; Lower the address of a load or a store. -(decl amode (Type Value u32) AMode) -;; TODO: Port lower_address() to ISLE. -(extern constructor amode amode) - (decl pair_amode (Value u32) PairAMode) (extern constructor pair_amode pair_amode) -;; Matches an `AMode` that is just a register. -(decl pure amode_is_reg (AMode) Reg) -;; TODO: Implement in ISLE. -(extern constructor amode_is_reg amode_is_reg) - ;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Helper for creating the zero register. (decl zero_reg () Reg) (extern constructor zero_reg zero_reg) +(decl fp_reg () Reg) +(extern constructor fp_reg fp_reg) + +(decl stack_reg () Reg) +(extern constructor stack_reg stack_reg) + (decl writable_zero_reg () WritableReg) (extern constructor writable_zero_reg writable_zero_reg) @@ -2615,14 +2718,33 @@ ;; Helper for emitting `MInst.LoadAddr` instructions. (decl load_addr (AMode) Reg) + +(rule (load_addr (AMode.UnsignedOffset r imm)) + (if (is_zero_uimm12 imm)) + r) + +(rule (load_addr (AMode.Unscaled r imm)) + (if (is_zero_simm9 imm)) + r) + +(rule (load_addr (AMode.RegOffset r 0 _)) r) +(rule (load_addr (AMode.FPOffset 0 _)) (fp_reg)) +(rule (load_addr (AMode.SPOffset 0 _)) (stack_reg)) + (rule (load_addr addr) (let ((dst WritableReg (temp_writable_reg $I64)) (_ Unit (emit (MInst.LoadAddr dst addr)))) dst)) -(rule (load_addr addr) - (if-let addr_reg (amode_is_reg addr)) - addr_reg) +;; Lower the address of a load or a store. +(decl amode (Type Value u32) AMode) +;; TODO: Port lower_address() to ISLE. +(extern constructor amode amode) + +(decl sink_load_into_amode (Type Inst) AMode) +(rule (sink_load_into_amode ty x @ (load _ addr offset)) + (let ((_ Unit (sink_inst x))) + (amode ty addr offset))) ;; Lower a constant f64. (decl constant_f64 (u64) Reg) diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs index 4428be2a8370..64906dd0232a 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/args.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs @@ -115,118 +115,81 @@ pub enum MemLabel { PCRel(i32), } -/// An addressing mode specified for a load/store operation. -#[derive(Clone, Debug)] -pub enum AMode { - // - // Real ARM64 addressing modes: - // - /// "post-indexed" mode as per AArch64 docs: postincrement reg after address computation. - PostIndexed(Writable, SImm9), - /// "pre-indexed" mode as per AArch64 docs: preincrement reg before address computation. - PreIndexed(Writable, SImm9), - - // N.B.: RegReg, RegScaled, and RegScaledExtended all correspond to - // what the ISA calls the "register offset" addressing mode. We split out - // several options here for more ergonomic codegen. - /// Register plus register offset. - RegReg(Reg, Reg), - - #[allow(dead_code)] - /// Register plus register offset, scaled by type's size. - RegScaled(Reg, Reg, Type), - - /// Register plus register offset, scaled by type's size, with index sign- or zero-extended - /// first. - RegScaledExtended(Reg, Reg, Type, ExtendOp), - - /// Register plus register offset, with index sign- or zero-extended first. - RegExtended(Reg, Reg, ExtendOp), - - /// Unscaled signed 9-bit immediate offset from reg. - Unscaled(Reg, SImm9), - - /// Scaled (by size of a type) unsigned 12-bit immediate offset from reg. - UnsignedOffset(Reg, UImm12Scaled), - - // - // virtual addressing modes that are lowered at emission time: - // - /// Reference to a "label": e.g., a symbol. - Label(MemLabel), - - /// Arbitrary offset from a register. Converted to generation of large - /// offsets with multiple instructions as necessary during code emission. - RegOffset(Reg, i64, Type), - - /// Offset from the stack pointer. - SPOffset(i64, Type), - - /// Offset from the frame pointer. - FPOffset(i64, Type), - - /// Offset from the "nominal stack pointer", which is where the real SP is - /// just after stack and spill slots are allocated in the function prologue. - /// At emission time, this is converted to `SPOffset` with a fixup added to - /// the offset constant. The fixup is a running value that is tracked as - /// emission iterates through instructions in linear order, and can be - /// adjusted up and down with [Inst::VirtualSPOffsetAdj]. - /// - /// The standard ABI is in charge of handling this (by emitting the - /// adjustment meta-instructions). It maintains the invariant that "nominal - /// SP" is where the actual SP is after the function prologue and before - /// clobber pushes. See the diagram in the documentation for - /// [crate::isa::aarch64::abi](the ABI module) for more details. - NominalSPOffset(i64, Type), -} - impl AMode { /// Memory reference using an address in a register. pub fn reg(reg: Reg) -> AMode { // Use UnsignedOffset rather than Unscaled to use ldr rather than ldur. // This also does not use PostIndexed / PreIndexed as they update the register. - AMode::UnsignedOffset(reg, UImm12Scaled::zero(I64)) + AMode::UnsignedOffset { + rn: reg, + uimm12: UImm12Scaled::zero(I64), + } } /// Memory reference using `reg1 + sizeof(ty) * reg2` as an address, with `reg2` sign- or /// zero-extended as per `op`. pub fn reg_plus_reg_scaled_extended(reg1: Reg, reg2: Reg, ty: Type, op: ExtendOp) -> AMode { - AMode::RegScaledExtended(reg1, reg2, ty, op) - } - - /// Does the address resolve to just a register value, with no offset or - /// other computation? - pub fn is_reg(&self) -> Option { - match self { - &AMode::UnsignedOffset(r, uimm12) if uimm12.value() == 0 => Some(r), - &AMode::Unscaled(r, imm9) if imm9.value() == 0 => Some(r), - &AMode::RegOffset(r, off, _) if off == 0 => Some(r), - &AMode::FPOffset(off, _) if off == 0 => Some(fp_reg()), - &AMode::SPOffset(off, _) if off == 0 => Some(stack_reg()), - _ => None, + AMode::RegScaledExtended { + rn: reg1, + rm: reg2, + ty, + extendop: op, } } pub fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self { // This should match `memarg_operands()`. match self { - &AMode::Unscaled(reg, imm9) => AMode::Unscaled(allocs.next(reg), imm9), - &AMode::UnsignedOffset(r, uimm12) => AMode::UnsignedOffset(allocs.next(r), uimm12), - &AMode::RegReg(r1, r2) => AMode::RegReg(allocs.next(r1), allocs.next(r2)), - &AMode::RegScaled(r1, r2, ty) => AMode::RegScaled(allocs.next(r1), allocs.next(r2), ty), - &AMode::RegScaledExtended(r1, r2, ty, ext) => { - AMode::RegScaledExtended(allocs.next(r1), allocs.next(r2), ty, ext) - } - &AMode::RegExtended(r1, r2, ext) => { - AMode::RegExtended(allocs.next(r1), allocs.next(r2), ext) - } - &AMode::PreIndexed(reg, simm9) => AMode::PreIndexed(allocs.next_writable(reg), simm9), - &AMode::PostIndexed(reg, simm9) => AMode::PostIndexed(allocs.next_writable(reg), simm9), - &AMode::RegOffset(r, off, ty) => AMode::RegOffset(allocs.next(r), off, ty), - &AMode::FPOffset(..) - | &AMode::SPOffset(..) - | &AMode::NominalSPOffset(..) - | AMode::Label(..) => self.clone(), + &AMode::Unscaled { rn, simm9 } => AMode::Unscaled { + rn: allocs.next(rn), + simm9, + }, + &AMode::UnsignedOffset { rn, uimm12 } => AMode::UnsignedOffset { + rn: allocs.next(rn), + uimm12, + }, + &AMode::RegReg { rn, rm } => AMode::RegReg { + rn: allocs.next(rn), + rm: allocs.next(rm), + }, + &AMode::RegScaled { rn, rm, ty } => AMode::RegScaled { + rn: allocs.next(rn), + rm: allocs.next(rm), + ty, + }, + &AMode::RegScaledExtended { + rn, + rm, + ty, + extendop, + } => AMode::RegScaledExtended { + rn: allocs.next(rn), + rm: allocs.next(rm), + ty, + extendop, + }, + &AMode::RegExtended { rn, rm, extendop } => AMode::RegExtended { + rn: allocs.next(rn), + rm: allocs.next(rm), + extendop, + }, + &AMode::PreIndexed { rn, simm9 } => AMode::PreIndexed { + rn: allocs.next_writable(rn), + simm9, + }, + &AMode::PostIndexed { rn, simm9 } => AMode::PostIndexed { + rn: allocs.next_writable(rn), + simm9, + }, + &AMode::RegOffset { rn, off, ty } => AMode::RegOffset { + rn: allocs.next(rn), + off, + ty, + }, + &AMode::FPOffset { .. } + | &AMode::SPOffset { .. } + | &AMode::NominalSPOffset { .. } + | AMode::Label { .. } => self.clone(), } } } @@ -419,8 +382,8 @@ fn shift_for_type(ty: Type) -> usize { impl PrettyPrint for AMode { fn pretty_print(&self, _: u8, allocs: &mut AllocationConsumer<'_>) -> String { match self { - &AMode::Unscaled(reg, simm9) => { - let reg = pretty_print_reg(reg, allocs); + &AMode::Unscaled { rn, simm9 } => { + let reg = pretty_print_reg(rn, allocs); if simm9.value != 0 { let simm9 = simm9.pretty_print(8, allocs); format!("[{}, {}]", reg, simm9) @@ -428,8 +391,8 @@ impl PrettyPrint for AMode { format!("[{}]", reg) } } - &AMode::UnsignedOffset(reg, uimm12) => { - let reg = pretty_print_reg(reg, allocs); + &AMode::UnsignedOffset { rn, uimm12 } => { + let reg = pretty_print_reg(rn, allocs); if uimm12.value != 0 { let uimm12 = uimm12.pretty_print(8, allocs); format!("[{}, {}]", reg, uimm12) @@ -437,55 +400,60 @@ impl PrettyPrint for AMode { format!("[{}]", reg) } } - &AMode::RegReg(r1, r2) => { - let r1 = pretty_print_reg(r1, allocs); - let r2 = pretty_print_reg(r2, allocs); + &AMode::RegReg { rn, rm } => { + let r1 = pretty_print_reg(rn, allocs); + let r2 = pretty_print_reg(rm, allocs); format!("[{}, {}]", r1, r2) } - &AMode::RegScaled(r1, r2, ty) => { - let r1 = pretty_print_reg(r1, allocs); - let r2 = pretty_print_reg(r2, allocs); + &AMode::RegScaled { rn, rm, ty } => { + let r1 = pretty_print_reg(rn, allocs); + let r2 = pretty_print_reg(rm, allocs); let shift = shift_for_type(ty); format!("[{}, {}, LSL #{}]", r1, r2, shift) } - &AMode::RegScaledExtended(r1, r2, ty, op) => { + &AMode::RegScaledExtended { + rn, + rm, + ty, + extendop, + } => { let shift = shift_for_type(ty); - let size = match op { + let size = match extendop { ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32, _ => OperandSize::Size64, }; - let r1 = pretty_print_reg(r1, allocs); - let r2 = pretty_print_ireg(r2, size, allocs); - let op = op.pretty_print(0, allocs); + let r1 = pretty_print_reg(rn, allocs); + let r2 = pretty_print_ireg(rm, size, allocs); + let op = extendop.pretty_print(0, allocs); format!("[{}, {}, {} #{}]", r1, r2, op, shift) } - &AMode::RegExtended(r1, r2, op) => { - let size = match op { + &AMode::RegExtended { rn, rm, extendop } => { + let size = match extendop { ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32, _ => OperandSize::Size64, }; - let r1 = pretty_print_reg(r1, allocs); - let r2 = pretty_print_ireg(r2, size, allocs); - let op = op.pretty_print(0, allocs); + let r1 = pretty_print_reg(rn, allocs); + let r2 = pretty_print_ireg(rm, size, allocs); + let op = extendop.pretty_print(0, allocs); format!("[{}, {}, {}]", r1, r2, op) } - &AMode::Label(ref label) => label.pretty_print(0, allocs), - &AMode::PreIndexed(r, simm9) => { - let r = pretty_print_reg(r.to_reg(), allocs); + &AMode::Label { ref label } => label.pretty_print(0, allocs), + &AMode::PreIndexed { rn, simm9 } => { + let r = pretty_print_reg(rn.to_reg(), allocs); let simm9 = simm9.pretty_print(8, allocs); format!("[{}, {}]!", r, simm9) } - &AMode::PostIndexed(r, simm9) => { - let r = pretty_print_reg(r.to_reg(), allocs); + &AMode::PostIndexed { rn, simm9 } => { + let r = pretty_print_reg(rn.to_reg(), allocs); let simm9 = simm9.pretty_print(8, allocs); format!("[{}], {}", r, simm9) } // Eliminated by `mem_finalize()`. - &AMode::SPOffset(..) - | &AMode::FPOffset(..) - | &AMode::NominalSPOffset(..) - | &AMode::RegOffset(..) => { - panic!("Unexpected pseudo mem-arg mode (stack-offset or generic reg-offset)!") + &AMode::SPOffset { .. } + | &AMode::FPOffset { .. } + | &AMode::NominalSPOffset { .. } + | &AMode::RegOffset { .. } => { + panic!("Unexpected pseudo mem-arg mode: {:?}", self) } } } diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 57e1bfb488e1..604e1d83c3a0 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -28,18 +28,18 @@ pub fn mem_finalize( state: &EmitState, ) -> (SmallVec<[Inst; 4]>, AMode) { match mem { - &AMode::RegOffset(_, off, ty) - | &AMode::SPOffset(off, ty) - | &AMode::FPOffset(off, ty) - | &AMode::NominalSPOffset(off, ty) => { + &AMode::RegOffset { off, ty, .. } + | &AMode::SPOffset { off, ty } + | &AMode::FPOffset { off, ty } + | &AMode::NominalSPOffset { off, ty } => { let basereg = match mem { - &AMode::RegOffset(reg, _, _) => reg, - &AMode::SPOffset(..) | &AMode::NominalSPOffset(..) => stack_reg(), - &AMode::FPOffset(..) => fp_reg(), + &AMode::RegOffset { rn, .. } => rn, + &AMode::SPOffset { .. } | &AMode::NominalSPOffset { .. } => stack_reg(), + &AMode::FPOffset { .. } => fp_reg(), _ => unreachable!(), }; let adj = match mem { - &AMode::NominalSPOffset(..) => { + &AMode::NominalSPOffset { .. } => { trace!( "mem_finalize: nominal SP offset {} + adj {} -> {}", off, @@ -53,10 +53,13 @@ pub fn mem_finalize( let off = off + adj; if let Some(simm9) = SImm9::maybe_from_i64(off) { - let mem = AMode::Unscaled(basereg, simm9); + let mem = AMode::Unscaled { rn: basereg, simm9 }; (smallvec![], mem) - } else if let Some(uimm12s) = UImm12Scaled::maybe_from_i64(off, ty) { - let mem = AMode::UnsignedOffset(basereg, uimm12s); + } else if let Some(uimm12) = UImm12Scaled::maybe_from_i64(off, ty) { + let mem = AMode::UnsignedOffset { + rn: basereg, + uimm12, + }; (smallvec![], mem) } else { let tmp = writable_spilltmp_reg(); @@ -78,9 +81,14 @@ pub fn mem_finalize( } } - &AMode::Label(ref label) => { + &AMode::Label { ref label } => { let off = memlabel_finalize(insn_off, label); - (smallvec![], AMode::Label(MemLabel::PCRel(off))) + ( + smallvec![], + AMode::Label { + label: MemLabel::PCRel(off), + }, + ) } _ => (smallvec![], mem.clone()), @@ -969,40 +977,41 @@ impl MachInstEmit for Inst { } match &mem { - &AMode::Unscaled(reg, simm9) => { - let reg = allocs.next(reg); + &AMode::Unscaled { rn, simm9 } => { + let reg = allocs.next(rn); sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd)); } - &AMode::UnsignedOffset(reg, uimm12scaled) => { - let reg = allocs.next(reg); - if uimm12scaled.value() != 0 { - assert_eq!(bits, ty_bits(uimm12scaled.scale_ty())); + &AMode::UnsignedOffset { rn, uimm12 } => { + let reg = allocs.next(rn); + if uimm12.value() != 0 { + assert_eq!(bits, ty_bits(uimm12.scale_ty())); } - sink.put4(enc_ldst_uimm12(op, uimm12scaled, reg, rd)); + sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd)); } - &AMode::RegReg(r1, r2) => { - let r1 = allocs.next(r1); - let r2 = allocs.next(r2); + &AMode::RegReg { rn, rm } => { + let r1 = allocs.next(rn); + let r2 = allocs.next(rm); sink.put4(enc_ldst_reg( op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd, )); } - &AMode::RegScaled(r1, r2, ty) | &AMode::RegScaledExtended(r1, r2, ty, _) => { - let r1 = allocs.next(r1); - let r2 = allocs.next(r2); + &AMode::RegScaled { rn, rm, ty } + | &AMode::RegScaledExtended { rn, rm, ty, .. } => { + let r1 = allocs.next(rn); + let r2 = allocs.next(rm); assert_eq!(bits, ty_bits(ty)); let extendop = match &mem { - &AMode::RegScaled(..) => None, - &AMode::RegScaledExtended(_, _, _, op) => Some(op), + &AMode::RegScaled { .. } => None, + &AMode::RegScaledExtended { extendop, .. } => Some(extendop), _ => unreachable!(), }; sink.put4(enc_ldst_reg( op, r1, r2, /* scaled = */ true, extendop, rd, )); } - &AMode::RegExtended(r1, r2, extendop) => { - let r1 = allocs.next(r1); - let r2 = allocs.next(r2); + &AMode::RegExtended { rn, rm, extendop } => { + let r1 = allocs.next(rn); + let r2 = allocs.next(rm); sink.put4(enc_ldst_reg( op, r1, @@ -1012,7 +1021,7 @@ impl MachInstEmit for Inst { rd, )); } - &AMode::Label(ref label) => { + &AMode::Label { ref label } => { let offset = match label { // cast i32 to u32 (two's-complement) &MemLabel::PCRel(off) => off as u32, @@ -1040,19 +1049,21 @@ impl MachInstEmit for Inst { _ => panic!("Unspported size for LDR from constant pool!"), } } - &AMode::PreIndexed(reg, simm9) => { - let reg = allocs.next(reg.to_reg()); + &AMode::PreIndexed { rn, simm9 } => { + let reg = allocs.next(rn.to_reg()); sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd)); } - &AMode::PostIndexed(reg, simm9) => { - let reg = allocs.next(reg.to_reg()); + &AMode::PostIndexed { rn, simm9 } => { + let reg = allocs.next(rn.to_reg()); sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd)); } // Eliminated by `mem_finalize()` above. - &AMode::SPOffset(..) | &AMode::FPOffset(..) | &AMode::NominalSPOffset(..) => { - panic!("Should not see stack-offset here!") + &AMode::SPOffset { .. } + | &AMode::FPOffset { .. } + | &AMode::NominalSPOffset { .. } + | &AMode::RegOffset { .. } => { + panic!("Should not see {:?} here!", mem) } - &AMode::RegOffset(..) => panic!("SHould not see generic reg-offset here!"), } } @@ -1089,39 +1100,39 @@ impl MachInstEmit for Inst { } match &mem { - &AMode::Unscaled(reg, simm9) => { - let reg = allocs.next(reg); + &AMode::Unscaled { rn, simm9 } => { + let reg = allocs.next(rn); sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd)); } - &AMode::UnsignedOffset(reg, uimm12scaled) => { - let reg = allocs.next(reg); - if uimm12scaled.value() != 0 { - assert_eq!(bits, ty_bits(uimm12scaled.scale_ty())); + &AMode::UnsignedOffset { rn, uimm12 } => { + let reg = allocs.next(rn); + if uimm12.value() != 0 { + assert_eq!(bits, ty_bits(uimm12.scale_ty())); } - sink.put4(enc_ldst_uimm12(op, uimm12scaled, reg, rd)); + sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd)); } - &AMode::RegReg(r1, r2) => { - let r1 = allocs.next(r1); - let r2 = allocs.next(r2); + &AMode::RegReg { rn, rm } => { + let r1 = allocs.next(rn); + let r2 = allocs.next(rm); sink.put4(enc_ldst_reg( op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd, )); } - &AMode::RegScaled(r1, r2, _ty) | &AMode::RegScaledExtended(r1, r2, _ty, _) => { - let r1 = allocs.next(r1); - let r2 = allocs.next(r2); + &AMode::RegScaled { rn, rm, .. } | &AMode::RegScaledExtended { rn, rm, .. } => { + let r1 = allocs.next(rn); + let r2 = allocs.next(rm); let extendop = match &mem { - &AMode::RegScaled(..) => None, - &AMode::RegScaledExtended(_, _, _, op) => Some(op), + &AMode::RegScaled { .. } => None, + &AMode::RegScaledExtended { extendop, .. } => Some(extendop), _ => unreachable!(), }; sink.put4(enc_ldst_reg( op, r1, r2, /* scaled = */ true, extendop, rd, )); } - &AMode::RegExtended(r1, r2, extendop) => { - let r1 = allocs.next(r1); - let r2 = allocs.next(r2); + &AMode::RegExtended { rn, rm, extendop } => { + let r1 = allocs.next(rn); + let r2 = allocs.next(rm); sink.put4(enc_ldst_reg( op, r1, @@ -1131,22 +1142,24 @@ impl MachInstEmit for Inst { rd, )); } - &AMode::Label(..) => { + &AMode::Label { .. } => { panic!("Store to a MemLabel not implemented!"); } - &AMode::PreIndexed(reg, simm9) => { - let reg = allocs.next(reg.to_reg()); + &AMode::PreIndexed { rn, simm9 } => { + let reg = allocs.next(rn.to_reg()); sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd)); } - &AMode::PostIndexed(reg, simm9) => { - let reg = allocs.next(reg.to_reg()); + &AMode::PostIndexed { rn, simm9 } => { + let reg = allocs.next(rn.to_reg()); sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd)); } // Eliminated by `mem_finalize()` above. - &AMode::SPOffset(..) | &AMode::FPOffset(..) | &AMode::NominalSPOffset(..) => { - panic!("Should not see stack-offset here!") + &AMode::SPOffset { .. } + | &AMode::FPOffset { .. } + | &AMode::NominalSPOffset { .. } + | &AMode::RegOffset { .. } => { + panic!("Should not see {:?} here!", mem) } - &AMode::RegOffset(..) => panic!("SHould not see generic reg-offset here!"), } } @@ -2176,7 +2189,9 @@ impl MachInstEmit for Inst { let rd = allocs.next_writable(rd); let inst = Inst::FpuLoad64 { rd, - mem: AMode::Label(MemLabel::PCRel(8)), + mem: AMode::Label { + label: MemLabel::PCRel(8), + }, flags: MemFlags::trusted(), }; inst.emit(&[], sink, emit_info, state); @@ -2190,7 +2205,9 @@ impl MachInstEmit for Inst { let rd = allocs.next_writable(rd); let inst = Inst::FpuLoad128 { rd, - mem: AMode::Label(MemLabel::PCRel(8)), + mem: AMode::Label { + label: MemLabel::PCRel(8), + }, flags: MemFlags::trusted(), }; inst.emit(&[], sink, emit_info, state); @@ -3069,7 +3086,9 @@ impl MachInstEmit for Inst { let rd = allocs.next_writable(rd); let inst = Inst::ULoad64 { rd, - mem: AMode::Label(MemLabel::PCRel(8)), + mem: AMode::Label { + label: MemLabel::PCRel(8), + }, flags: MemFlags::trusted(), }; inst.emit(&[], sink, emit_info, state); @@ -3089,17 +3108,17 @@ impl MachInstEmit for Inst { } let (reg, index_reg, offset) = match mem { - AMode::RegExtended(r, idx, extendop) => { - let r = allocs.next(r); - (r, Some((idx, extendop)), 0) + AMode::RegExtended { rn, rm, extendop } => { + let r = allocs.next(rn); + (r, Some((rm, extendop)), 0) } - AMode::Unscaled(r, simm9) => { - let r = allocs.next(r); + AMode::Unscaled { rn, simm9 } => { + let r = allocs.next(rn); (r, None, simm9.value()) } - AMode::UnsignedOffset(r, uimm12scaled) => { - let r = allocs.next(r); - (r, None, uimm12scaled.value() as i32) + AMode::UnsignedOffset { rn, uimm12 } => { + let r = allocs.next(rn); + (r, None, uimm12.value() as i32) } _ => panic!("Unsupported case for LoadAddr: {:?}", mem), }; diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index b3dc56d568f4..f496c77a8c8a 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -1371,7 +1371,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad8 { rd: writable_xreg(1), - mem: AMode::Unscaled(xreg(2), simm9_zero()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: simm9_zero(), + }, flags: MemFlags::trusted(), }, "41004038", @@ -1380,7 +1383,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad8 { rd: writable_xreg(1), - mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::zero(I8)), + mem: AMode::UnsignedOffset { + rn: xreg(2), + uimm12: UImm12Scaled::zero(I8), + }, flags: MemFlags::trusted(), }, "41004039", @@ -1389,7 +1395,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad8 { rd: writable_xreg(1), - mem: AMode::RegReg(xreg(2), xreg(5)), + mem: AMode::RegReg { + rn: xreg(2), + rm: xreg(5), + }, flags: MemFlags::trusted(), }, "41686538", @@ -1398,7 +1407,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::SLoad8 { rd: writable_xreg(1), - mem: AMode::Unscaled(xreg(2), simm9_zero()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: simm9_zero(), + }, flags: MemFlags::trusted(), }, "41008038", @@ -1407,7 +1419,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::SLoad8 { rd: writable_xreg(1), - mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(63, I8).unwrap()), + mem: AMode::UnsignedOffset { + rn: xreg(2), + uimm12: UImm12Scaled::maybe_from_i64(63, I8).unwrap(), + }, flags: MemFlags::trusted(), }, "41FC8039", @@ -1416,7 +1431,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::SLoad8 { rd: writable_xreg(1), - mem: AMode::RegReg(xreg(2), xreg(5)), + mem: AMode::RegReg { + rn: xreg(2), + rm: xreg(5), + }, flags: MemFlags::trusted(), }, "4168A538", @@ -1425,7 +1443,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad16 { rd: writable_xreg(1), - mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(5).unwrap()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: SImm9::maybe_from_i64(5).unwrap(), + }, flags: MemFlags::trusted(), }, "41504078", @@ -1434,7 +1455,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad16 { rd: writable_xreg(1), - mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8, I16).unwrap()), + mem: AMode::UnsignedOffset { + rn: xreg(2), + uimm12: UImm12Scaled::maybe_from_i64(8, I16).unwrap(), + }, flags: MemFlags::trusted(), }, "41104079", @@ -1443,7 +1467,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad16 { rd: writable_xreg(1), - mem: AMode::RegScaled(xreg(2), xreg(3), I16), + mem: AMode::RegScaled { + rn: xreg(2), + rm: xreg(3), + ty: I16, + }, flags: MemFlags::trusted(), }, "41786378", @@ -1452,7 +1480,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::SLoad16 { rd: writable_xreg(1), - mem: AMode::Unscaled(xreg(2), simm9_zero()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: simm9_zero(), + }, flags: MemFlags::trusted(), }, "41008078", @@ -1461,7 +1492,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::SLoad16 { rd: writable_xreg(28), - mem: AMode::UnsignedOffset(xreg(20), UImm12Scaled::maybe_from_i64(24, I16).unwrap()), + mem: AMode::UnsignedOffset { + rn: xreg(20), + uimm12: UImm12Scaled::maybe_from_i64(24, I16).unwrap(), + }, flags: MemFlags::trusted(), }, "9C328079", @@ -1470,7 +1504,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::SLoad16 { rd: writable_xreg(28), - mem: AMode::RegScaled(xreg(20), xreg(20), I16), + mem: AMode::RegScaled { + rn: xreg(20), + rm: xreg(20), + ty: I16, + }, flags: MemFlags::trusted(), }, "9C7AB478", @@ -1479,7 +1517,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad32 { rd: writable_xreg(1), - mem: AMode::Unscaled(xreg(2), simm9_zero()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: simm9_zero(), + }, flags: MemFlags::trusted(), }, "410040B8", @@ -1488,7 +1529,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad32 { rd: writable_xreg(12), - mem: AMode::UnsignedOffset(xreg(0), UImm12Scaled::maybe_from_i64(204, I32).unwrap()), + mem: AMode::UnsignedOffset { + rn: xreg(0), + uimm12: UImm12Scaled::maybe_from_i64(204, I32).unwrap(), + }, flags: MemFlags::trusted(), }, "0CCC40B9", @@ -1497,7 +1541,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad32 { rd: writable_xreg(1), - mem: AMode::RegScaled(xreg(2), xreg(12), I32), + mem: AMode::RegScaled { + rn: xreg(2), + rm: xreg(12), + ty: I32, + }, flags: MemFlags::trusted(), }, "41786CB8", @@ -1506,7 +1554,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::SLoad32 { rd: writable_xreg(1), - mem: AMode::Unscaled(xreg(2), simm9_zero()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: simm9_zero(), + }, flags: MemFlags::trusted(), }, "410080B8", @@ -1515,7 +1566,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::SLoad32 { rd: writable_xreg(12), - mem: AMode::UnsignedOffset(xreg(1), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()), + mem: AMode::UnsignedOffset { + rn: xreg(1), + uimm12: UImm12Scaled::maybe_from_i64(16380, I32).unwrap(), + }, flags: MemFlags::trusted(), }, "2CFCBFB9", @@ -1524,7 +1578,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::SLoad32 { rd: writable_xreg(1), - mem: AMode::RegScaled(xreg(5), xreg(1), I32), + mem: AMode::RegScaled { + rn: xreg(5), + rm: xreg(1), + ty: I32, + }, flags: MemFlags::trusted(), }, "A178A1B8", @@ -1533,7 +1591,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::Unscaled(xreg(2), simm9_zero()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: simm9_zero(), + }, flags: MemFlags::trusted(), }, "410040F8", @@ -1542,7 +1603,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(-256).unwrap()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: SImm9::maybe_from_i64(-256).unwrap(), + }, flags: MemFlags::trusted(), }, "410050F8", @@ -1551,7 +1615,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(255).unwrap()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: SImm9::maybe_from_i64(255).unwrap(), + }, flags: MemFlags::trusted(), }, "41F04FF8", @@ -1560,7 +1627,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()), + mem: AMode::UnsignedOffset { + rn: xreg(2), + uimm12: UImm12Scaled::maybe_from_i64(32760, I64).unwrap(), + }, flags: MemFlags::trusted(), }, "41FC7FF9", @@ -1569,7 +1639,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::RegReg(xreg(2), xreg(3)), + mem: AMode::RegReg { + rn: xreg(2), + rm: xreg(3), + }, flags: MemFlags::trusted(), }, "416863F8", @@ -1578,7 +1651,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::RegScaled(xreg(2), xreg(3), I64), + mem: AMode::RegScaled { + rn: xreg(2), + rm: xreg(3), + ty: I64, + }, flags: MemFlags::trusted(), }, "417863F8", @@ -1587,7 +1664,12 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::SXTW), + mem: AMode::RegScaledExtended { + rn: xreg(2), + rm: xreg(3), + ty: I64, + extendop: ExtendOp::SXTW, + }, flags: MemFlags::trusted(), }, "41D863F8", @@ -1596,7 +1678,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::SXTW), + mem: AMode::RegExtended { + rn: xreg(2), + rm: xreg(3), + extendop: ExtendOp::SXTW, + }, flags: MemFlags::trusted(), }, "41C863F8", @@ -1605,7 +1691,9 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::Label(MemLabel::PCRel(64)), + mem: AMode::Label { + label: MemLabel::PCRel(64), + }, flags: MemFlags::trusted(), }, "01020058", @@ -1614,7 +1702,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), + mem: AMode::PreIndexed { + rn: writable_xreg(2), + simm9: SImm9::maybe_from_i64(16).unwrap(), + }, flags: MemFlags::trusted(), }, "410C41F8", @@ -1623,7 +1714,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), + mem: AMode::PostIndexed { + rn: writable_xreg(2), + simm9: SImm9::maybe_from_i64(16).unwrap(), + }, flags: MemFlags::trusted(), }, "410441F8", @@ -1632,7 +1726,7 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::FPOffset(32768, I8), + mem: AMode::FPOffset { off: 32768, ty: I8 }, flags: MemFlags::trusted(), }, "100090D2B063308B010240F9", @@ -1641,7 +1735,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::FPOffset(-32768, I8), + mem: AMode::FPOffset { + off: -32768, + ty: I8, + }, flags: MemFlags::trusted(), }, "F0FF8F92B063308B010240F9", @@ -1650,7 +1747,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::FPOffset(1048576, I8), // 2^20 + mem: AMode::FPOffset { + off: 1048576, + ty: I8, + }, // 2^20 flags: MemFlags::trusted(), }, "1002A0D2B063308B010240F9", @@ -1659,7 +1759,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::FPOffset(1048576 + 1, I8), // 2^20 + 1 + mem: AMode::FPOffset { + off: 1048576 + 1, + ty: I8, + }, // 2^20 + 1 flags: MemFlags::trusted(), }, "300080521002A072B063308B010240F9", @@ -1669,7 +1772,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::RegOffset(xreg(7), 8, I64), + mem: AMode::RegOffset { + rn: xreg(7), + off: 8, + ty: I64, + }, flags: MemFlags::trusted(), }, "E18040F8", @@ -1679,7 +1786,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::RegOffset(xreg(7), 1024, I64), + mem: AMode::RegOffset { + rn: xreg(7), + off: 1024, + ty: I64, + }, flags: MemFlags::trusted(), }, "E10042F9", @@ -1689,7 +1800,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::ULoad64 { rd: writable_xreg(1), - mem: AMode::RegOffset(xreg(7), 1048576, I64), + mem: AMode::RegOffset { + rn: xreg(7), + off: 1048576, + ty: I64, + }, flags: MemFlags::trusted(), }, "1002A0D2F060308B010240F9", @@ -1699,7 +1814,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store8 { rd: xreg(1), - mem: AMode::Unscaled(xreg(2), simm9_zero()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: simm9_zero(), + }, flags: MemFlags::trusted(), }, "41000038", @@ -1708,7 +1826,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store8 { rd: xreg(1), - mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(4095, I8).unwrap()), + mem: AMode::UnsignedOffset { + rn: xreg(2), + uimm12: UImm12Scaled::maybe_from_i64(4095, I8).unwrap(), + }, flags: MemFlags::trusted(), }, "41FC3F39", @@ -1717,7 +1838,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store16 { rd: xreg(1), - mem: AMode::Unscaled(xreg(2), simm9_zero()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: simm9_zero(), + }, flags: MemFlags::trusted(), }, "41000078", @@ -1726,7 +1850,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store16 { rd: xreg(1), - mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8190, I16).unwrap()), + mem: AMode::UnsignedOffset { + rn: xreg(2), + uimm12: UImm12Scaled::maybe_from_i64(8190, I16).unwrap(), + }, flags: MemFlags::trusted(), }, "41FC3F79", @@ -1735,7 +1862,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store32 { rd: xreg(1), - mem: AMode::Unscaled(xreg(2), simm9_zero()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: simm9_zero(), + }, flags: MemFlags::trusted(), }, "410000B8", @@ -1744,7 +1874,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store32 { rd: xreg(1), - mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()), + mem: AMode::UnsignedOffset { + rn: xreg(2), + uimm12: UImm12Scaled::maybe_from_i64(16380, I32).unwrap(), + }, flags: MemFlags::trusted(), }, "41FC3FB9", @@ -1753,7 +1886,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store64 { rd: xreg(1), - mem: AMode::Unscaled(xreg(2), simm9_zero()), + mem: AMode::Unscaled { + rn: xreg(2), + simm9: simm9_zero(), + }, flags: MemFlags::trusted(), }, "410000F8", @@ -1762,7 +1898,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store64 { rd: xreg(1), - mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()), + mem: AMode::UnsignedOffset { + rn: xreg(2), + uimm12: UImm12Scaled::maybe_from_i64(32760, I64).unwrap(), + }, flags: MemFlags::trusted(), }, "41FC3FF9", @@ -1771,7 +1910,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store64 { rd: xreg(1), - mem: AMode::RegReg(xreg(2), xreg(3)), + mem: AMode::RegReg { + rn: xreg(2), + rm: xreg(3), + }, flags: MemFlags::trusted(), }, "416823F8", @@ -1780,7 +1922,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store64 { rd: xreg(1), - mem: AMode::RegScaled(xreg(2), xreg(3), I64), + mem: AMode::RegScaled { + rn: xreg(2), + rm: xreg(3), + ty: I64, + }, flags: MemFlags::trusted(), }, "417823F8", @@ -1789,7 +1935,12 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store64 { rd: xreg(1), - mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::UXTW), + mem: AMode::RegScaledExtended { + rn: xreg(2), + rm: xreg(3), + ty: I64, + extendop: ExtendOp::UXTW, + }, flags: MemFlags::trusted(), }, "415823F8", @@ -1798,7 +1949,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store64 { rd: xreg(1), - mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::UXTW), + mem: AMode::RegExtended { + rn: xreg(2), + rm: xreg(3), + extendop: ExtendOp::UXTW, + }, flags: MemFlags::trusted(), }, "414823F8", @@ -1807,7 +1962,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store64 { rd: xreg(1), - mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), + mem: AMode::PreIndexed { + rn: writable_xreg(2), + simm9: SImm9::maybe_from_i64(16).unwrap(), + }, flags: MemFlags::trusted(), }, "410C01F8", @@ -1816,7 +1974,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Store64 { rd: xreg(1), - mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), + mem: AMode::PostIndexed { + rn: writable_xreg(2), + simm9: SImm9::maybe_from_i64(16).unwrap(), + }, flags: MemFlags::trusted(), }, "410401F8", @@ -6403,7 +6564,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::FpuLoad32 { rd: writable_vreg(16), - mem: AMode::RegScaled(xreg(8), xreg(9), F32), + mem: AMode::RegScaled { + rn: xreg(8), + rm: xreg(9), + ty: F32, + }, flags: MemFlags::trusted(), }, "107969BC", @@ -6413,7 +6578,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::FpuLoad64 { rd: writable_vreg(16), - mem: AMode::RegScaled(xreg(8), xreg(9), F64), + mem: AMode::RegScaled { + rn: xreg(8), + rm: xreg(9), + ty: F64, + }, flags: MemFlags::trusted(), }, "107969FC", @@ -6423,7 +6592,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::FpuLoad128 { rd: writable_vreg(16), - mem: AMode::RegScaled(xreg(8), xreg(9), I128), + mem: AMode::RegScaled { + rn: xreg(8), + rm: xreg(9), + ty: I128, + }, flags: MemFlags::trusted(), }, "1079E93C", @@ -6433,7 +6606,9 @@ fn test_aarch64_binemit() { insns.push(( Inst::FpuLoad32 { rd: writable_vreg(16), - mem: AMode::Label(MemLabel::PCRel(8)), + mem: AMode::Label { + label: MemLabel::PCRel(8), + }, flags: MemFlags::trusted(), }, "5000001C", @@ -6443,7 +6618,9 @@ fn test_aarch64_binemit() { insns.push(( Inst::FpuLoad64 { rd: writable_vreg(16), - mem: AMode::Label(MemLabel::PCRel(8)), + mem: AMode::Label { + label: MemLabel::PCRel(8), + }, flags: MemFlags::trusted(), }, "5000005C", @@ -6453,7 +6630,9 @@ fn test_aarch64_binemit() { insns.push(( Inst::FpuLoad128 { rd: writable_vreg(16), - mem: AMode::Label(MemLabel::PCRel(8)), + mem: AMode::Label { + label: MemLabel::PCRel(8), + }, flags: MemFlags::trusted(), }, "5000009C", @@ -6463,7 +6642,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::FpuStore32 { rd: vreg(16), - mem: AMode::RegScaled(xreg(8), xreg(9), F32), + mem: AMode::RegScaled { + rn: xreg(8), + rm: xreg(9), + ty: F32, + }, flags: MemFlags::trusted(), }, "107929BC", @@ -6473,7 +6656,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::FpuStore64 { rd: vreg(16), - mem: AMode::RegScaled(xreg(8), xreg(9), F64), + mem: AMode::RegScaled { + rn: xreg(8), + rm: xreg(9), + ty: F64, + }, flags: MemFlags::trusted(), }, "107929FC", @@ -6483,7 +6670,11 @@ fn test_aarch64_binemit() { insns.push(( Inst::FpuStore128 { rd: vreg(16), - mem: AMode::RegScaled(xreg(8), xreg(9), I128), + mem: AMode::RegScaled { + rn: xreg(8), + rm: xreg(9), + ty: I128, + }, flags: MemFlags::trusted(), }, "1079A93C", diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 3d55806cf5f4..01b66914390a 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -36,7 +36,7 @@ mod emit_tests; // Instructions (top level): definition pub use crate::isa::aarch64::lower::isle::generated_code::{ - ALUOp, ALUOp3, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BitOp, FPUOp1, FPUOp2, FPUOp3, + ALUOp, ALUOp3, AMode, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BitOp, FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp, VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp, VecRRNarrowOp, VecRRPairLongOp, VecRRRLongOp, VecShiftImmModOp, VecShiftImmOp, @@ -539,24 +539,24 @@ impl Inst { fn memarg_operands VReg>(memarg: &AMode, collector: &mut OperandCollector<'_, F>) { // This should match `AMode::with_allocs()`. match memarg { - &AMode::Unscaled(reg, ..) | &AMode::UnsignedOffset(reg, ..) => { - collector.reg_use(reg); + &AMode::Unscaled { rn, .. } | &AMode::UnsignedOffset { rn, .. } => { + collector.reg_use(rn); } - &AMode::RegReg(r1, r2, ..) - | &AMode::RegScaled(r1, r2, ..) - | &AMode::RegScaledExtended(r1, r2, ..) - | &AMode::RegExtended(r1, r2, ..) => { - collector.reg_use(r1); - collector.reg_use(r2); + &AMode::RegReg { rn, rm, .. } + | &AMode::RegScaled { rn, rm, .. } + | &AMode::RegScaledExtended { rn, rm, .. } + | &AMode::RegExtended { rn, rm, .. } => { + collector.reg_use(rn); + collector.reg_use(rm); } - &AMode::Label(..) => {} - &AMode::PreIndexed(reg, ..) | &AMode::PostIndexed(reg, ..) => { - collector.reg_mod(reg); + &AMode::Label { .. } => {} + &AMode::PreIndexed { rn, .. } | &AMode::PostIndexed { rn, .. } => { + collector.reg_mod(rn); } - &AMode::FPOffset(..) => {} - &AMode::SPOffset(..) | &AMode::NominalSPOffset(..) => {} - &AMode::RegOffset(r, ..) => { - collector.reg_use(r); + &AMode::FPOffset { .. } => {} + &AMode::SPOffset { .. } | &AMode::NominalSPOffset { .. } => {} + &AMode::RegOffset { rn, .. } => { + collector.reg_use(rn); } } } @@ -1416,7 +1416,7 @@ impl Inst { | &Inst::SLoad32 { rd, ref mem, .. } | &Inst::ULoad64 { rd, ref mem, .. } => { let is_unscaled = match &mem { - &AMode::Unscaled(..) => true, + &AMode::Unscaled { .. } => true, _ => false, }; let (op, size) = match (self, is_unscaled) { @@ -1449,7 +1449,7 @@ impl Inst { | &Inst::Store32 { rd, ref mem, .. } | &Inst::Store64 { rd, ref mem, .. } => { let is_unscaled = match &mem { - &AMode::Unscaled(..) => true, + &AMode::Unscaled { .. } => true, _ => false, }; let (op, size) = match (self, is_unscaled) { @@ -2632,11 +2632,9 @@ impl Inst { ); } let (reg, index_reg, offset) = match mem { - AMode::RegExtended(r, idx, extendop) => (r, Some((idx, extendop)), 0), - AMode::Unscaled(r, simm9) => (r, None, simm9.value()), - AMode::UnsignedOffset(r, uimm12scaled) => { - (r, None, uimm12scaled.value() as i32) - } + AMode::RegExtended { rn, rm, extendop } => (rn, Some((rm, extendop)), 0), + AMode::Unscaled { rn, simm9 } => (rn, None, simm9.value()), + AMode::UnsignedOffset { rn, uimm12 } => (rn, None, uimm12.value() as i32), _ => panic!("Unsupported case for LoadAddr: {:?}", mem), }; let abs_offset = if offset < 0 { diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index b0bb4e6f1cd2..0398d0028073 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -1772,12 +1772,11 @@ (rule (lower (has_type ty (splat (ireduce (iconst (u64_from_imm64 n)))))) (splat_const n (vector_size ty))) -(rule (lower (has_type ty (splat x @ (load flags addr offset)))) +(rule (lower (has_type ty (splat x @ (load flags _ _)))) (if-let mem_op (is_sinkable_inst x)) - (let ((_ Unit (sink_inst mem_op)) - (addr AMode (amode (lane_type ty) addr offset)) - (address Reg (load_addr addr))) - (ld1r address (vector_size ty) flags))) + (let ((addr AMode (sink_load_into_amode (lane_type ty) mem_op)) + (address Reg (load_addr addr))) + (ld1r address (vector_size ty) flags))) ;;;; Rules for `AtomicLoad` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (valid_atomic_transaction ty) (atomic_load flags addr))) diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index 7921290a1d34..3c69ad0c9eff 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -658,16 +658,24 @@ pub(crate) fn lower_address( if addends32.len() > 0 { let (reg32, extendop) = addends32.pop().unwrap(); let reg64 = addends64.pop().unwrap(); - AMode::RegExtended(reg64, reg32, extendop) + AMode::RegExtended { + rn: reg64, + rm: reg32, + extendop, + } } else if offset > 0 && offset < 0x1000 { let reg64 = addends64.pop().unwrap(); let off = offset; offset = 0; - AMode::RegOffset(reg64, off, elem_ty) + AMode::RegOffset { + rn: reg64, + off, + ty: elem_ty, + } } else if addends64.len() >= 2 { let reg1 = addends64.pop().unwrap(); let reg2 = addends64.pop().unwrap(); - AMode::RegReg(reg1, reg2) + AMode::RegReg { rn: reg1, rm: reg2 } } else { let reg1 = addends64.pop().unwrap(); AMode::reg(reg1) @@ -691,7 +699,11 @@ pub(crate) fn lower_address( to_bits: 64, }); if let Some((reg2, extendop)) = addends32.pop() { - AMode::RegExtended(tmp.to_reg(), reg2, extendop) + AMode::RegExtended { + rn: tmp.to_reg(), + rm: reg2, + extendop, + } } else { AMode::reg(tmp.to_reg()) } @@ -716,12 +728,36 @@ pub(crate) fn lower_address( // Allocate the temp and shoehorn it into the AMode. let addr = ctx.alloc_tmp(I64).only_reg().unwrap(); let (reg, memarg) = match memarg { - AMode::RegExtended(r1, r2, extendop) => { - (r1, AMode::RegExtended(addr.to_reg(), r2, extendop)) - } - AMode::RegOffset(r, off, ty) => (r, AMode::RegOffset(addr.to_reg(), off, ty)), - AMode::RegReg(r1, r2) => (r2, AMode::RegReg(addr.to_reg(), r1)), - AMode::UnsignedOffset(r, imm) => (r, AMode::UnsignedOffset(addr.to_reg(), imm)), + AMode::RegExtended { rn, rm, extendop } => ( + rn, + AMode::RegExtended { + rn: addr.to_reg(), + rm, + extendop, + }, + ), + AMode::RegOffset { rn, off, ty } => ( + rn, + AMode::RegOffset { + rn: addr.to_reg(), + off, + ty, + }, + ), + AMode::RegReg { rn, rm } => ( + rm, + AMode::RegReg { + rn: addr.to_reg(), + rm: rn, + }, + ), + AMode::UnsignedOffset { rn, uimm12 } => ( + rn, + AMode::UnsignedOffset { + rn: addr.to_reg(), + uimm12, + }, + ), _ => unreachable!(), }; diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 231011bd9303..e8c3b26f47f0 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -6,12 +6,12 @@ use generated_code::Context; // Types that the generated ISLE code uses via `use super::*`. use super::{ - lower_constant_f128, lower_constant_f32, lower_constant_f64, lower_fp_condcode, - writable_zero_reg, zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, BranchTarget, CallIndInfo, - CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, FloatCC, Imm12, ImmLogic, ImmShift, - Inst as MInst, IntCC, JTSequenceInfo, MachLabel, MoveWideConst, MoveWideOp, NarrowValueMode, - Opcode, OperandSize, PairAMode, Reg, ScalarSize, ShiftOpAndAmt, UImm5, VecMisc2, VectorSize, - NZCV, + fp_reg, lower_constant_f128, lower_constant_f32, lower_constant_f64, lower_fp_condcode, + stack_reg, writable_zero_reg, zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, BranchTarget, + CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, FloatCC, Imm12, ImmLogic, ImmShift, + Inst as MInst, IntCC, JTSequenceInfo, MachLabel, MemLabel, MoveWideConst, MoveWideOp, + NarrowValueMode, Opcode, OperandSize, PairAMode, Reg, SImm9, ScalarSize, ShiftOpAndAmt, + UImm12Scaled, UImm5, VecMisc2, VectorSize, NZCV, }; use crate::ir::condcodes; use crate::isa::aarch64::inst::{FPULeftShiftImm, FPURightShiftImm}; @@ -152,6 +152,22 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { } } + fn is_zero_simm9(&mut self, imm: &SImm9) -> Option<()> { + if imm.value() == 0 { + Some(()) + } else { + None + } + } + + fn is_zero_uimm12(&mut self, imm: &UImm12Scaled) -> Option<()> { + if imm.value() == 0 { + Some(()) + } else { + None + } + } + /// This is target-word-size dependent. And it excludes booleans and reftypes. fn valid_atomic_transaction(&mut self, ty: Type) -> Option { match ty { @@ -294,6 +310,14 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { zero_reg() } + fn stack_reg(&mut self) -> Reg { + stack_reg() + } + + fn fp_reg(&mut self) -> Reg { + fp_reg() + } + fn xreg(&mut self, index: u8) -> Reg { xreg(index) } @@ -494,10 +518,6 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { lower_pair_address(self.lower_ctx, addr, offset as i32) } - fn amode_is_reg(&mut self, address: &AMode) -> Option { - address.is_reg() - } - fn constant_f64(&mut self, value: u64) -> Reg { let rd = self.temp_writable_reg(I8X16);