AArch64: port load and store operations to ISLE.

This retains `lower_amode` in the handwritten code (@akirilov-arm
reports that there is an upcoming patch to port this), but tweaks it
slightly to take a `Value` rather than an `Inst`.

cranelift/codegen/src/ir/dynamic_type.rs

@@ -1,6 +1,7 @@
 //! Dynamic IR types
 
 use crate::ir::entities::DynamicType;
+use crate::ir::types::*;
 use crate::ir::GlobalValue;
 use crate::ir::PrimaryMap;
 use crate::ir::Type;
@@ -36,3 +37,19 @@ impl DynamicTypeData {
 
 /// All allocated dynamic types.
 pub type DynamicTypes = PrimaryMap<DynamicType, DynamicTypeData>;
+
+/// Convert a dynamic-vector type to a fixed-vector type.
+pub fn dynamic_to_fixed(ty: Type) -> Type {
+    match ty {
+        I8X8XN => I8X8,
+        I8X16XN => I8X16,
+        I16X4XN => I16X4,
+        I16X8XN => I16X8,
+        I32X2XN => I32X2,
+        I32X4XN => I32X4,
+        I64X2XN => I64X2,
+        F32X4XN => F32X4,
+        F64X2XN => F64X2,
+        _ => unreachable!("unhandled type: {}", ty),
+    }
+}

cranelift/codegen/src/ir/mod.rs

@@ -35,7 +35,7 @@ pub use crate::ir::builder::{
 };
 pub use crate::ir::constant::{ConstantData, ConstantPool};
 pub use crate::ir::dfg::{DataFlowGraph, ValueDef};
-pub use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes};
+pub use crate::ir::dynamic_type::{dynamic_to_fixed, DynamicTypeData, DynamicTypes};
 pub use crate::ir::entities::{
     Block, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Heap, Immediate, Inst,
     JumpTable, SigRef, StackSlot, Table, UserExternalNameRef, Value,

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -5,7 +5,7 @@ use crate::ir::types;
 use crate::ir::types::*;
 use crate::ir::MemFlags;
 use crate::ir::Opcode;
-use crate::ir::{ExternalName, LibCall, Signature};
+use crate::ir::{dynamic_to_fixed, ExternalName, LibCall, Signature};
 use crate::isa;
 use crate::isa::aarch64::{inst::EmitState, inst::*, settings as aarch64_settings};
 use crate::isa::unwind::UnwindInst;

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -1502,10 +1502,13 @@
 (extern constructor cond_br_cond cond_br_cond)
 
 ;; Lower the address of a load or a store.
-(decl amode (Type Inst u32) AMode)
+(decl amode (Type Value u32) AMode)
 ;; TODO: Port lower_address() to ISLE.
 (extern constructor amode amode)
 
+(decl pair_amode (Value u32) PairAMode)
+(extern constructor pair_amode pair_amode)
+
 ;; Matches an `AMode` that is just a register.
 (decl pure amode_is_reg (AMode) Reg)
 ;; TODO: Implement in ISLE.
@@ -2337,6 +2340,92 @@
 (rule (udf trap_code)
       (SideEffectNoResult.Inst (MInst.Udf trap_code)))
 
+;; Helpers for generating various load instructions, with varying
+;; widths and sign/zero-extending properties.
+(decl aarch64_uload8 (AMode MemFlags) Reg)
+(rule (aarch64_uload8 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.ULoad8 dst amode flags))))
+        dst))
+(decl aarch64_sload8 (AMode MemFlags) Reg)
+(rule (aarch64_sload8 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.SLoad8 dst amode flags))))
+        dst))
+(decl aarch64_uload16 (AMode MemFlags) Reg)
+(rule (aarch64_uload16 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.ULoad16 dst amode flags))))
+        dst))
+(decl aarch64_sload16 (AMode MemFlags) Reg)
+(rule (aarch64_sload16 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.SLoad16 dst amode flags))))
+        dst))
+(decl aarch64_uload32 (AMode MemFlags) Reg)
+(rule (aarch64_uload32 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.ULoad32 dst amode flags))))
+        dst))
+(decl aarch64_sload32 (AMode MemFlags) Reg)
+(rule (aarch64_sload32 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.SLoad32 dst amode flags))))
+        dst))
+(decl aarch64_uload64 (AMode MemFlags) Reg)
+(rule (aarch64_uload64 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.ULoad64 dst amode flags))))
+        dst))
+(decl aarch64_fpuload32 (AMode MemFlags) Reg)
+(rule (aarch64_fpuload32 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $F64))
+            (_ Unit (emit (MInst.FpuLoad32 dst amode flags))))
+        dst))
+(decl aarch64_fpuload64 (AMode MemFlags) Reg)
+(rule (aarch64_fpuload64 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $F64))
+            (_ Unit (emit (MInst.FpuLoad64 dst amode flags))))
+        dst))
+(decl aarch64_fpuload128 (AMode MemFlags) Reg)
+(rule (aarch64_fpuload128 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $F64X2))
+            (_ Unit (emit (MInst.FpuLoad128 dst amode flags))))
+        dst))
+(decl aarch64_loadp64 (PairAMode MemFlags) ValueRegs)
+(rule (aarch64_loadp64 amode flags)
+      (let ((dst1 WritableReg (temp_writable_reg $I64))
+            (dst2 WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.LoadP64 dst1 dst2 amode flags))))
+        (value_regs dst1 dst2)))
+
+;; Helpers for generating various store instructions with varying
+;; widths.
+(decl aarch64_store8 (AMode MemFlags Reg) SideEffectNoResult)
+(rule (aarch64_store8 amode flags val)
+      (SideEffectNoResult.Inst (MInst.Store8 val amode flags)))
+(decl aarch64_store16 (AMode MemFlags Reg) SideEffectNoResult)
+(rule (aarch64_store16 amode flags val)
+      (SideEffectNoResult.Inst (MInst.Store16 val amode flags)))
+(decl aarch64_store32 (AMode MemFlags Reg) SideEffectNoResult)
+(rule (aarch64_store32 amode flags val)
+      (SideEffectNoResult.Inst (MInst.Store32 val amode flags)))
+(decl aarch64_store64 (AMode MemFlags Reg) SideEffectNoResult)
+(rule (aarch64_store64 amode flags val)
+      (SideEffectNoResult.Inst (MInst.Store64 val amode flags)))
+(decl aarch64_fpustore32 (AMode MemFlags Reg) SideEffectNoResult)
+(rule (aarch64_fpustore32 amode flags val)
+      (SideEffectNoResult.Inst (MInst.FpuStore32 val amode flags)))
+(decl aarch64_fpustore64 (AMode MemFlags Reg) SideEffectNoResult)
+(rule (aarch64_fpustore64 amode flags val)
+      (SideEffectNoResult.Inst (MInst.FpuStore64 val amode flags)))
+(decl aarch64_fpustore128 (AMode MemFlags Reg) SideEffectNoResult)
+(rule (aarch64_fpustore128 amode flags val)
+      (SideEffectNoResult.Inst (MInst.FpuStore128 val amode flags)))
+(decl aarch64_storep64 (PairAMode MemFlags Reg Reg) SideEffectNoResult)
+(rule (aarch64_storep64 amode flags val1 val2)
+      (SideEffectNoResult.Inst (MInst.StoreP64 val1 val2 amode flags)))
+
 ;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Type of extension performed by an immediate helper

cranelift/codegen/src/isa/aarch64/inst/args.rs

@@ -773,18 +773,3 @@ impl VectorSize {
         }
     }
 }
-
-pub(crate) fn dynamic_to_fixed(ty: Type) -> Type {
-    match ty {
-        I8X8XN => I8X8,
-        I8X16XN => I8X16,
-        I16X4XN => I16X4,
-        I16X8XN => I16X8,
-        I32X2XN => I32X2,
-        I32X4XN => I32X4,
-        I64X2XN => I64X2,
-        F32X4XN => F32X4,
-        F64X2XN => F64X2,
-        _ => unreachable!("unhandled type: {}", ty),
-    }
-}

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -1777,10 +1777,10 @@
 (rule (lower (has_type ty (splat (ireduce (iconst (u64_from_imm64 n))))))
       (splat_const n (vector_size ty)))
 
-(rule (lower (has_type ty (splat x @ (load flags _addr offset))))
+(rule (lower (has_type ty (splat x @ (load flags addr offset))))
       (if-let mem_op (is_sinkable_inst x))
       (let ((_ Unit (sink_inst mem_op))
-            (addr AMode (amode (lane_type ty) mem_op offset))
+            (addr AMode (amode (lane_type ty) addr offset))
             (address Reg (load_addr addr)))
            (ld1r address (vector_size ty) flags)))
 
@@ -2031,6 +2031,174 @@
 (rule (lower (return args))
       (lower_return (range 0 (value_slice_len args)) args))
 
+;;;; Rules for loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower
+       (has_type $I8 (load flags address offset)))
+      (aarch64_uload8 (amode $I8 address offset) flags))
+(rule (lower
+       (has_type $I16 (load flags address offset)))
+      (aarch64_uload16 (amode $I16 address offset) flags))
+(rule (lower
+       (has_type $I32 (load flags address offset)))
+      (aarch64_uload32 (amode $I32 address offset) flags))
+(rule (lower
+       (has_type $I64 (load flags address offset)))
+      (aarch64_uload64 (amode $I64 address offset) flags))
+(rule (lower
+       (has_type $R64 (load flags address offset)))
+      (aarch64_uload64 (amode $I64 address offset) flags))
+(rule (lower
+       (has_type $F32 (load flags address offset)))
+      (aarch64_fpuload32 (amode $F32 address offset) flags))
+(rule (lower
+       (has_type $F64 (load flags address offset)))
+      (aarch64_fpuload64 (amode $F64 address offset) flags))
+(rule (lower
+       (has_type $I128 (load flags address offset)))
+      (aarch64_loadp64 (pair_amode address offset) flags))
+(rule (lower
+       (has_type (ty_vec64 _)
+                        (load flags address offset)))
+      (aarch64_fpuload128 (amode $F64 address offset) flags))
+(rule (lower
+       (has_type (ty_vec128 _)
+                        (load flags address offset)))
+      (aarch64_fpuload128 (amode $I8X16 address offset) flags))
+(rule (lower
+       (has_type (ty_dyn_vec64 _)
+                        (load flags address offset)))
+      (aarch64_fpuload64 (amode $F64 address offset) flags))
+(rule (lower
+       (has_type (ty_dyn_vec128 _)
+                        (load flags address offset)))
+      (aarch64_fpuload128 (amode $I8X16 address offset) flags))
+
+(rule (lower
+       (uload8 flags address offset))
+      (aarch64_uload8 (amode $I8 address offset) flags))
+(rule (lower
+       (sload8 flags address offset))
+      (aarch64_sload8 (amode $I8 address offset) flags))
+(rule (lower
+       (uload16 flags address offset))
+      (aarch64_uload16 (amode $I16 address offset) flags))
+(rule (lower
+       (sload16 flags address offset))
+      (aarch64_sload16 (amode $I16 address offset) flags))
+(rule (lower
+       (uload32 flags address offset))
+      (aarch64_uload32 (amode $I32 address offset) flags))
+(rule (lower
+       (sload32 flags address offset))
+      (aarch64_sload32 (amode $I32 address offset) flags))
+
+(rule (lower
+       (sload8x8 flags address offset))
+      (vec_extend (VecExtendOp.Sxtl)
+                  (aarch64_fpuload64 (amode $F64 address offset) flags)
+                  $false
+                  (ScalarSize.Size16)))
+(rule (lower
+       (uload8x8 flags address offset))
+      (vec_extend (VecExtendOp.Uxtl)
+                  (aarch64_fpuload64 (amode $F64 address offset) flags)
+                  $false
+                  (ScalarSize.Size16)))
+(rule (lower
+       (sload16x4 flags address offset))
+      (vec_extend (VecExtendOp.Sxtl)
+                  (aarch64_fpuload64 (amode $F64 address offset) flags)
+                  $false
+                  (ScalarSize.Size32)))
+(rule (lower
+       (uload16x4 flags address offset))
+      (vec_extend (VecExtendOp.Uxtl)
+                  (aarch64_fpuload64 (amode $F64 address offset) flags)
+                  $false
+                  (ScalarSize.Size32)))
+(rule (lower
+       (sload32x2 flags address offset))
+      (vec_extend (VecExtendOp.Sxtl)
+                  (aarch64_fpuload64 (amode $F64 address offset) flags)
+                  $false
+                  (ScalarSize.Size64)))
+(rule (lower
+       (uload32x2 flags address offset))
+      (vec_extend (VecExtendOp.Uxtl)
+                  (aarch64_fpuload64 (amode $F64 address offset) flags)
+                  $false
+                  (ScalarSize.Size64)))
+
+;;;; Rules for stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower
+       (store flags value @ (value_type $I8) address offset))
+      (side_effect
+       (aarch64_store8 (amode $I8 address offset) flags value)))
+(rule (lower
+       (store flags value @ (value_type $I16) address offset))
+      (side_effect
+       (aarch64_store16 (amode $I16 address offset) flags value)))
+(rule (lower
+       (store flags value @ (value_type $I32) address offset))
+      (side_effect
+       (aarch64_store32 (amode $I32 address offset) flags value)))
+(rule (lower
+       (store flags value @ (value_type $I64) address offset))
+      (side_effect
+       (aarch64_store64 (amode $I64 address offset) flags value)))
+(rule (lower
+       (store flags value @ (value_type $R64) address offset))
+      (side_effect
+       (aarch64_store64 (amode $I64 address offset) flags value)))
+
+(rule (lower
+       (istore8 flags value address offset))
+      (side_effect
+       (aarch64_store8 (amode $I8 address offset) flags value)))
+(rule (lower
+       (istore16 flags value address offset))
+      (side_effect
+       (aarch64_store16 (amode $I16 address offset) flags value)))
+(rule (lower
+       (istore32 flags value address offset))
+      (side_effect
+       (aarch64_store32 (amode $I32 address offset) flags value)))
+
+(rule (lower
+       (store flags value @ (value_type $F32) address offset))
+      (side_effect
+       (aarch64_fpustore32 (amode $F32 address offset) flags value)))
+(rule (lower
+       (store flags value @ (value_type $F64) address offset))
+      (side_effect
+       (aarch64_fpustore64 (amode $F64 address offset) flags value)))
+
+(rule (lower
+       (store flags value @ (value_type $I128) address offset))
+      (side_effect
+       (aarch64_storep64 (pair_amode address offset) flags
+                         (value_regs_get value 0)
+                         (value_regs_get value 1))))
+
+(rule (lower
+       (store flags value @ (value_type (ty_vec64 _)) address offset))
+      (side_effect
+       (aarch64_fpustore64 (amode $F64 address offset) flags value)))
+(rule (lower
+       (store flags value @ (value_type (ty_vec128 _)) address offset))
+      (side_effect
+       (aarch64_fpustore128 (amode $I8X16 address offset) flags value)))
+(rule (lower
+       (store flags value @ (value_type (ty_dyn_vec64 _)) address offset))
+      (side_effect
+       (aarch64_fpustore64 (amode $F64 address offset) flags value)))
+(rule (lower
+       (store flags value @ (value_type (ty_dyn_vec128 _)) address offset))
+      (side_effect
+       (aarch64_fpustore128 (amode $I8X16 address offset) flags value)))
+
 ;;; Rules for `{get,set}_pinned_reg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (get_pinned_reg))