Port flag-based ops to ISLE (AArch64)

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -2077,6 +2077,14 @@
  (MInst.FpuCSel64 dst if_true if_false cond)
  dst)))
 
+;; Helper for emitting `MInst.VecCSel` instructions.
+(decl vec_csel (Cond Reg Reg) ConsumesFlags)
+(rule (vec_csel cond if_true if_false)
+ (let ((dst WritableReg (temp_writable_reg $I8X16)))
+ (ConsumesFlags.ConsumesFlagsReturnsReg
+ (MInst.VecCSel dst if_true if_false cond)
+ dst)))
+
 ;; Helper for emitting `MInst.FpuRound` instructions.
 (decl fpu_round (FpuRoundMode Reg) Reg)
 (rule (fpu_round op rn)
@@ -2353,6 +2361,11 @@
 (rule (aarch64_fence)
  (SideEffectNoResult.Inst (MInst.Fence)))
 
+;; Helper for generating `csdb` instructions.
+(decl csdb () SideEffectNoResult)
+(rule (csdb)
+ (SideEffectNoResult.Inst (MInst.Csdb)))
+
 ;; Helper for generating `brk` instructions.
 (decl brk () SideEffectNoResult)
 (rule (brk)
@@ -2641,6 +2654,15 @@
 (rule (aarch64_storep64 amode flags val1 val2)
  (SideEffectNoResult.Inst (MInst.StoreP64 val1 val2 amode flags)))
 
+;; Helper for generating a `trapif` instruction.
+
+(decl trap_if (ProducesFlags TrapCode Cond) InstOutput)
+(rule (trap_if flags trap_code cond)
+ (side_effect
+ (with_flags_side_effect flags
+ (ConsumesFlags.ConsumesFlagsSideEffect
+ (MInst.TrapIf (cond_br_cond cond) trap_code)))))
+
 ;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Type of extension performed by an immediate helper
@@ -3378,7 +3400,9 @@
 ;; ccmp lhs_hi, rhs_hi, #0, eq
 (decl lower_icmp_i128_eq_ne (Value Value) ProducesFlags)
 (rule (lower_icmp_i128_eq_ne lhs rhs)
- (let ((lhs_lo Reg (value_regs_get lhs 0))
+ (let ((lhs ValueRegs (put_in_regs lhs))
+ (rhs ValueRegs (put_in_regs rhs))
+ (lhs_lo Reg (value_regs_get lhs 0))
  (lhs_hi Reg (value_regs_get lhs 1))
  (rhs_lo Reg (value_regs_get rhs 0))
  (rhs_hi Reg (value_regs_get rhs 1))
@@ -3399,6 +3423,8 @@
 (rule -1 (lower_icmp_into_reg cond lhs rhs $I128 out_ty)
  (let ((unsigned_cond Cond (cond_code (intcc_unsigned cond)))
  (cond Cond (cond_code cond))
+ (lhs ValueRegs (put_in_regs lhs))
+ (rhs ValueRegs (put_in_regs rhs))
  (lhs_lo Reg (value_regs_get lhs 0))
  (lhs_hi Reg (value_regs_get lhs 1))
  (rhs_lo Reg (value_regs_get rhs 0))
@@ -3477,3 +3503,30 @@
  (let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
  (dst Reg (value_regs_get dst 0)))
  (cmp (OperandSize.Size64) (zero_reg) dst)))
+
+;; Helpers for generating select instruction sequences.
+(decl lower_select (ProducesFlags Cond Type Value Value) ValueRegs)
+(rule (lower_select flags cond (ty_scalar_float ty) rn rm)
+ (with_flags flags (fpu_csel ty cond rn rm)))
+(rule (lower_select flags cond (ty_vec128 ty) rn rm)
+ (with_flags flags (vec_csel cond rn rm)))
+(rule (lower_select flags cond ty rn rm)
+ (if (ty_vec64 ty))
+ (with_flags flags (fpu_csel $F64 cond rn rm)))
+(rule (lower_select flags cond $I128 rn rm)
+ (let ((dst_lo WritableReg (temp_writable_reg $I64))
+ (dst_hi WritableReg (temp_writable_reg $I64))
+ (rn ValueRegs (put_in_regs rn))
+ (rm ValueRegs (put_in_regs rm))
+ (rn_lo Reg (value_regs_get rn 0))
+ (rn_hi Reg (value_regs_get rn 1))
+ (rm_lo Reg (value_regs_get rm 0))
+ (rm_hi Reg (value_regs_get rm 1)))
+ (with_flags flags
+ (ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
+ (MInst.CSel dst_lo cond rn_lo rm_lo)
+ (MInst.CSel dst_hi cond rn_hi rm_hi)
+ (value_regs dst_lo dst_hi)))))
+(rule (lower_select flags cond ty rn rm)
+ (if (ty_int_bool_ref_scalar_64 ty))
+ (with_flags flags (csel cond rn rm)))

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -1738,11 +1738,111 @@
 (rule (lower (trap trap_code))
  (side_effect (udf trap_code)))
 
+;;;; Rules for `trapif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (trapif cc insn @ (iadd_ifcout x y) trap_code))
+ ;; The flags must not have been clobbered by any other instruction, as
+ ;; verified by the CLIF validator; so we can simply use the flags here.
+ (let ((insn ProducesFlags (flags_to_producesflags insn)))
+ (trap_if insn trap_code (cond_code cc))))
+
+;; Verification ensures the input is always a single-def ifcmp.
+(rule (lower (trapif cc insn @ (ifcmp x @ (value_type ty) y) trap_code))
+ (let ((cond Cond (cond_code cc)))
+ (trap_if (lower_icmp_into_flags cc x y ty) trap_code cond)))
+
+;;;; Rules for `trapff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Verification ensures the input is always a single-def ffcmp.
+(rule (lower (trapff cc insn @ (ffcmp x @ (value_type ty) y) trap_code))
+ (let ((cond Cond (fp_cond_code cc)))
+ (trap_if (fpu_cmp (scalar_size ty) x y) trap_code cond)))
+
 ;;;; Rules for `resumable_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (resumable_trap trap_code))
  (side_effect (udf trap_code)))
 
+;;;; Rules for `trueif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Verification ensures the input is always a single-def ifcmp.
+(rule (lower (has_type out_ty
+ (trueif cc insn @ (ifcmp x @ (value_type in_ty) y))))
+ (lower_icmp_into_reg cc x y in_ty out_ty))
+
+;;;; Rules for `trueff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Verification ensures the input is always a single-def ffcmp.
+(rule (lower (has_type ty (trueff cc insn @ (ffcmp x @ (value_type in_ty) y))))
+ (with_flags_reg
+ (fpu_cmp (scalar_size in_ty) x y)
+ (materialize_bool_result (ty_bits ty) (fp_cond_code cc))))
+
+;;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type ty
+ (select _flags @ (icmp cc x @ (value_type in_ty) y) rn rm)))
+ (let ((cond Cond (cond_code cc)))
+ (lower_select
+ (lower_icmp_into_flags cc x y in_ty)
+ cond ty rn rm)))
+
+(rule (lower (has_type ty
+ (select _flags @ (bint (icmp cc x @ (value_type in_ty) y)) rn rm)))
+ (let ((cond Cond (cond_code cc)))
+ (lower_select
+ (lower_icmp_into_flags cc x y in_ty)
+ cond ty rn rm)))
+
+(rule (lower (has_type ty
+ (select _flags @ (fcmp cc x @ (value_type in_ty) y) rn rm)))
+ (let ((cond Cond (fp_cond_code cc)))
+ (lower_select
+ (fpu_cmp (scalar_size in_ty) x y)
+ cond ty rn rm)))
+
+(rule (lower (has_type ty
+ (select _flags @ (bint (fcmp cc x @ (value_type in_ty) y)) rn rm)))
+ (let ((cond Cond (fp_cond_code cc)))
+ (lower_select
+ (fpu_cmp (scalar_size in_ty) x y)
+ cond ty rn rm)))
+
+(rule -1 (lower (has_type ty (select rcond @ (value_type (fits_in_32 _)) rn rm)))
+ (let ((rcond Reg (put_in_reg_zext32 rcond)))
+ (lower_select
+ (cmp (OperandSize.Size32) rcond (zero_reg))
+ (Cond.Ne) ty rn rm)))
+
+(rule -2 (lower (has_type ty (select rcond rn rm)))
+ (let ((rcond Reg (put_in_reg_zext64 rcond)))
+ (lower_select
+ (cmp (OperandSize.Size64) rcond (zero_reg))
+ (Cond.Ne) ty rn rm)))
+
+;;;; Rules for `selectif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Verification ensures that the input is always a single-def ifcmp.
+(rule (lower (has_type ty
+ (selectif cc flags @ (ifcmp x @ (value_type in_ty) y)
+ if_true if_false)))
+ (let ((cond Cond (cond_code cc)))
+ (lower_select
+ (lower_icmp_into_flags cc x y in_ty)
+ cond ty if_true if_false)))
+
+;;;; Rules for `selectif_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type ty
+ (selectif_spectre_guard cc
+ flags @ (ifcmp x @ (value_type in_ty) y) if_true if_false)))
+ (let ((cond Cond (cond_code cc))
+ (dst ValueRegs (lower_select
+ (lower_icmp_into_flags cc x y in_ty)
+ cond ty if_true if_false))
+ (_ InstOutput (side_effect (csdb))))
+ dst))
+
 ;;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (has_type (ty_vec128 _) (vconst (u128_from_constant x))))
@@ -2395,18 +2495,6 @@
 (rule (lower (fvpromote_low val))
  (vec_rr_long (VecRRLongOp.Fcvtl32) val $false))
 
-;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; TODO: requires icmp/fcmp first.
-
-;;; Rules for `selectif` / `selectif_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; TODO: requires icmp/fcmp first.
-
-;;; Rules for `trueif` / `trueff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; TODO: requires icmp/fcmp first.
-
 ;;; Rules for `brz`/`brnz`/`brif`/`brff`/`bricmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; TODO: requires icmp/fcmp first.

cranelift/codegen/src/isa/aarch64/lower.rs

@@ -1178,15 +1178,12 @@ pub(crate) enum IcmpOutput {
  /// Lowers the comparison into a cond code, discarding the results. The cond code emitted can
  /// be checked in the resulting [IcmpResult].
  CondCode,
- /// Materializes the results into a register. This may overwrite any flags previously set.
- Register(Writable<Reg>),
 }
 
 impl IcmpOutput {
  pub fn reg(&self) -> Option<Writable<Reg>> {
  match self {
  IcmpOutput::CondCode => None,
- IcmpOutput::Register(reg) => Some(*reg),
  }
  }
 }
@@ -1197,15 +1194,12 @@ pub(crate) enum IcmpResult {
  /// The result was output into the given [Cond]. Callers may perform operations using this [Cond]
  /// and its inverse, other [Cond]'s are not guaranteed to be correct.
  CondCode(Cond),
- /// The result was materialized into the output register.
- Register,
 }
 
 impl IcmpResult {
  pub fn unwrap_cond(&self) -> Cond {
  match self {
  IcmpResult::CondCode(c) => *c,
- _ => panic!("Unwrapped cond, but IcmpResult was {:?}", self),
  }
  }
 }
@@ -1364,8 +1358,6 @@ pub(crate) fn lower_icmp(
  }
 
  Ok(match output {
- // We currently never emit a different register than what was asked for
- IcmpOutput::Register(_) => IcmpResult::Register,
  IcmpOutput::CondCode => IcmpResult::CondCode(out_condcode),
  })
 }