AArch64: port load operations to ISLE.

cranelift/codegen/src/ir/dynamic_type.rs

@@ -1,6 +1,7 @@
 //! Dynamic IR types
 
 use crate::ir::entities::DynamicType;
+use crate::ir::types::*;
 use crate::ir::GlobalValue;
 use crate::ir::PrimaryMap;
 use crate::ir::Type;
@@ -36,3 +37,19 @@ impl DynamicTypeData {
 
 /// All allocated dynamic types.
 pub type DynamicTypes = PrimaryMap<DynamicType, DynamicTypeData>;
+
+/// Convert a dynamic-vector type to a fixed-vector type.
+pub fn dynamic_to_fixed(ty: Type) -> Type {
+ match ty {
+ I8X8XN => I8X8,
+ I8X16XN => I8X16,
+ I16X4XN => I16X4,
+ I16X8XN => I16X8,
+ I32X2XN => I32X2,
+ I32X4XN => I32X4,
+ I64X2XN => I64X2,
+ F32X4XN => F32X4,
+ F64X2XN => F64X2,
+ _ => unreachable!("unhandled type: {}", ty),
+ }
+}

cranelift/codegen/src/ir/mod.rs

@@ -35,7 +35,7 @@ pub use crate::ir::builder::{
 };
 pub use crate::ir::constant::{ConstantData, ConstantPool};
 pub use crate::ir::dfg::{DataFlowGraph, ValueDef};
-pub use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes};
+pub use crate::ir::dynamic_type::{dynamic_to_fixed, DynamicTypeData, DynamicTypes};
 pub use crate::ir::entities::{
  Block, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Heap, Immediate, Inst,
  JumpTable, SigRef, StackSlot, Table, UserExternalNameRef, Value,

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -5,7 +5,7 @@ use crate::ir::types;
 use crate::ir::types::*;
 use crate::ir::MemFlags;
 use crate::ir::Opcode;
-use crate::ir::{ExternalName, LibCall, Signature};
+use crate::ir::{dynamic_to_fixed, ExternalName, LibCall, Signature};
 use crate::isa;
 use crate::isa::aarch64::{inst::EmitState, inst::*, settings as aarch64_settings};
 use crate::isa::unwind::UnwindInst;

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -1492,9 +1492,15 @@
 
 ;; Lower the address of a load or a store.
 (decl amode (Type Inst u32) AMode)
-;; TODO: Port lower_address() to ISLE.
+;; TODO: Port lower_address() to ISLE. We should take a single `Value`
+;; here, not an `Inst`; complex loads/stores with multiple addends
+;; directly on the instruction are legalized into loads/stores with
+;; only one address input.
 (extern constructor amode amode)
 
+(decl pair_amode (Inst u32) PairAMode)
+(extern constructor pair_amode pair_amode)
+
 ;; Matches an `AMode` that is just a register.
 (decl pure amode_is_reg (AMode) Reg)
 ;; TODO: Implement in ISLE.
@@ -2307,6 +2313,65 @@
 (rule (udf trap_code)
  (SideEffectNoResult.Inst (MInst.Udf trap_code)))
 
+;; Helpers for generating various load instructions, with varying
+;; widths and sign/zero-extending properties.
+(decl aarch64_uload8 (AMode MemFlags) Reg)
+(rule (aarch64_uload8 amode flags)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.ULoad8 dst amode flags))))
+ dst))
+(decl aarch64_sload8 (AMode MemFlags) Reg)
+(rule (aarch64_sload8 amode flags)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.SLoad8 dst amode flags))))
+ dst))
+(decl aarch64_uload16 (AMode MemFlags) Reg)
+(rule (aarch64_uload16 amode flags)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.ULoad16 dst amode flags))))
+ dst))
+(decl aarch64_sload16 (AMode MemFlags) Reg)
+(rule (aarch64_sload16 amode flags)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.SLoad16 dst amode flags))))
+ dst))
+(decl aarch64_uload32 (AMode MemFlags) Reg)
+(rule (aarch64_uload32 amode flags)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.ULoad32 dst amode flags))))
+ dst))
+(decl aarch64_sload32 (AMode MemFlags) Reg)
+(rule (aarch64_sload32 amode flags)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.SLoad32 dst amode flags))))
+ dst))
+(decl aarch64_uload64 (AMode MemFlags) Reg)
+(rule (aarch64_uload64 amode flags)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.ULoad64 dst amode flags))))
+ dst))
+(decl aarch64_fpuload32 (AMode MemFlags) Reg)
+(rule (aarch64_fpuload32 amode flags)
+ (let ((dst WritableReg (temp_writable_reg $F64))
+ (_ Unit (emit (MInst.FpuLoad32 dst amode flags))))
+ dst))
+(decl aarch64_fpuload64 (AMode MemFlags) Reg)
+(rule (aarch64_fpuload64 amode flags)
+ (let ((dst WritableReg (temp_writable_reg $F64))
+ (_ Unit (emit (MInst.FpuLoad64 dst amode flags))))
+ dst))
+(decl aarch64_fpuload128 (AMode MemFlags) Reg)
+(rule (aarch64_fpuload128 amode flags)
+ (let ((dst WritableReg (temp_writable_reg $F64X2))
+ (_ Unit (emit (MInst.FpuLoad128 dst amode flags))))
+ dst))
+(decl aarch64_loadp64 (PairAMode MemFlags) ValueRegs)
+(rule (aarch64_loadp64 amode flags)
+ (let ((dst1 WritableReg (temp_writable_reg $I64))
+ (dst2 WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.LoadP64 dst1 dst2 amode flags))))
+ (value_regs dst1 dst2)))
+
 ;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Type of extension performed by an immediate helper

cranelift/codegen/src/isa/aarch64/inst/args.rs

@@ -773,18 +773,3 @@ impl VectorSize {
  }
  }
 }
-
-pub(crate) fn dynamic_to_fixed(ty: Type) -> Type {
- match ty {
- I8X8XN => I8X8,
- I8X16XN => I8X16,
- I16X4XN => I16X4,
- I16X8XN => I16X8,
- I32X2XN => I32X2,
- I32X4XN => I32X4,
- I64X2XN => I64X2,
- F32X4XN => F32X4,
- F64X2XN => F64X2,
- _ => unreachable!("unhandled type: {}", ty),
- }
-}

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -2016,3 +2016,102 @@
 
 (rule (lower (get_return_address))
  (aarch64_link))
+
+;;;; Rules for loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower
+ inst @ (has_type $I8 (load flags address offset)))
+ (aarch64_uload8 (amode $I8 inst offset) flags))
+(rule (lower
+ inst @ (has_type $I16 (load flags address offset)))
+ (aarch64_uload16 (amode $I16 inst offset) flags))
+(rule (lower
+ inst @ (has_type $I32 (load flags address offset)))
+ (aarch64_uload32 (amode $I32 inst offset) flags))
+(rule (lower
+ inst @ (has_type $I64 (load flags address offset)))
+ (aarch64_uload64 (amode $I64 inst offset) flags))
+(rule (lower
+ inst @ (has_type $R64 (load flags address offset)))
+ (aarch64_uload64 (amode $I64 inst offset) flags))
+(rule (lower
+ inst @ (has_type $F32 (load flags address offset)))
+ (aarch64_fpuload32 (amode $F32 inst offset) flags))
+(rule (lower
+ inst @ (has_type $F64 (load flags address offset)))
+ (aarch64_fpuload64 (amode $F64 inst offset) flags))
+(rule (lower
+ inst @ (has_type $I128 (load flags address offset)))
+ (aarch64_loadp64 (pair_amode inst offset) flags))
+(rule (lower
+ inst @ (has_type (ty_vec64 _)
+ (load flags address offset)))
+ (aarch64_fpuload128 (amode $F64 inst offset) flags))
+(rule (lower
+ inst @ (has_type (ty_vec128 _)
+ (load flags address offset)))
+ (aarch64_fpuload128 (amode $I8X16 inst offset) flags))
+(rule (lower
+ inst @ (has_type (ty_dyn_vec64 _)
+ (load flags address offset)))
+ (aarch64_fpuload64 (amode $F64 inst offset) flags))
+(rule (lower
+ inst @ (has_type (ty_dyn_vec128 _)
+ (load flags address offset)))
+ (aarch64_fpuload128 (amode $I8X16 inst offset) flags))
+
+(rule (lower
+ inst @ (uload8 flags address offset))
+ (aarch64_uload8 (amode $I8 inst offset) flags))
+(rule (lower
+ inst @ (sload8 flags address offset))
+ (aarch64_sload8 (amode $I8 inst offset) flags))
+(rule (lower
+ inst @ (uload16 flags address offset))
+ (aarch64_uload16 (amode $I16 inst offset) flags))
+(rule (lower
+ inst @ (sload16 flags address offset))
+ (aarch64_sload16 (amode $I16 inst offset) flags))
+(rule (lower
+ inst @ (uload32 flags address offset))
+ (aarch64_uload32 (amode $I32 inst offset) flags))
+(rule (lower
+ inst @ (sload32 flags address offset))
+ (aarch64_sload32 (amode $I32 inst offset) flags))
+
+(rule (lower
+ inst @ (sload8x8 flags address offset))
+ (vec_extend (VecExtendOp.Sxtl)
+ (aarch64_fpuload64 (amode $F64 inst offset) flags)
+ $false
+ (ScalarSize.Size16)))
+(rule (lower
+ inst @ (uload8x8 flags address offset))
+ (vec_extend (VecExtendOp.Uxtl)
+ (aarch64_fpuload64 (amode $F64 inst offset) flags)
+ $false
+ (ScalarSize.Size16)))
+(rule (lower
+ inst @ (sload16x4 flags address offset))
+ (vec_extend (VecExtendOp.Sxtl)
+ (aarch64_fpuload64 (amode $F64 inst offset) flags)
+ $false
+ (ScalarSize.Size32)))
+(rule (lower
+ inst @ (uload16x4 flags address offset))
+ (vec_extend (VecExtendOp.Uxtl)
+ (aarch64_fpuload64 (amode $F64 inst offset) flags)
+ $false
+ (ScalarSize.Size32)))
+(rule (lower
+ inst @ (sload32x2 flags address offset))
+ (vec_extend (VecExtendOp.Sxtl)
+ (aarch64_fpuload64 (amode $F64 inst offset) flags)
+ $false
+ (ScalarSize.Size64)))
+(rule (lower
+ inst @ (uload32x2 flags address offset))
+ (vec_extend (VecExtendOp.Uxtl)
+ (aarch64_fpuload64 (amode $F64 inst offset) flags)
+ $false
+ (ScalarSize.Size64)))

cranelift/codegen/src/isa/aarch64/lower.rs

@@ -1452,41 +1452,6 @@ pub(crate) fn materialize_bool_result(
  }
 }
 
-fn load_op_to_ty(op: Opcode) -> Option<Type> {
- match op {
- Opcode::Sload8 | Opcode::Uload8 => Some(I8),
- Opcode::Sload16 | Opcode::Uload16 => Some(I16),
- Opcode::Sload32 | Opcode::Uload32 => Some(I32),
- Opcode::Load => None,
- Opcode::Sload8x8 | Opcode::Uload8x8 => Some(I8X8),
- Opcode::Sload16x4 | Opcode::Uload16x4 => Some(I16X4),
- Opcode::Sload32x2 | Opcode::Uload32x2 => Some(I32X2),
- _ => None,
- }
-}
-
-/// Helper to lower a load instruction; this is used in several places, because
-/// a load can sometimes be merged into another operation.
-pub(crate) fn lower_load<
- F: FnMut(&mut Lower<Inst>, ValueRegs<Writable<Reg>>, Type, AMode) -> CodegenResult<()>,
->(
- ctx: &mut Lower<Inst>,
- ir_inst: IRInst,
- inputs: &[InsnInput],
- output: InsnOutput,
- mut f: F,
-) -> CodegenResult<()> {
- let op = ctx.data(ir_inst).opcode();
-
- let elem_ty = load_op_to_ty(op).unwrap_or_else(|| ctx.output_ty(ir_inst, 0));
-
- let off = ctx.data(ir_inst).load_store_offset().unwrap();
- let mem = lower_address(ctx, elem_ty, &inputs[..], off);
- let rd = get_output_reg(ctx, output);
-
- f(ctx, rd, elem_ty, mem)
-}
-
 //=============================================================================
 // Lowering-backend trait implementation.
 

cranelift/codegen/src/isa/aarch64/lower/isle.rs

@@ -12,7 +12,7 @@ use super::{
  PairAMode, Reg, ScalarSize, ShiftOpAndAmt, UImm5, VecMisc2, VectorSize, NZCV,
 };
 use crate::isa::aarch64::inst::{FPULeftShiftImm, FPURightShiftImm};
-use crate::isa::aarch64::lower::{lower_address, lower_splat_const};
+use crate::isa::aarch64::lower::{lower_address, lower_pair_address, lower_splat_const};
 use crate::isa::aarch64::settings::Flags as IsaFlags;
 use crate::machinst::{isle::*, InputSourceInst};
 use crate::settings::Flags;
@@ -481,6 +481,14 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
  )
  }
 
+ fn pair_amode(&mut self, mem_op: Inst, offset: u32) -> PairAMode {
+ lower_pair_address(
+ self.lower_ctx,
+ &insn_inputs(self.lower_ctx, mem_op)[..],
+ offset as i32,
+ )
+ }
+
  fn amode_is_reg(&mut self, address: &AMode) -> Option<Reg> {
  address.is_reg()
  }