x64: port select using an FP comparison to ISLE

This change includes quite a few interlocking parts, required mainly by
the current x64 conventions in ISLE:
 - it adds a way to emit a `cmove` with multiple OR-ing conditions;
   because x64 ISLE cannot currently safely emit a comparison followed
   by several jumps, this adds `MachInst::CmoveOr` and
   `MachInst::XmmCmoveOr` macro instructions. Unfortunately, these macro
   instructions hide the multi-instruction sequence in `lower.isle`
 - to properly keep track of what instructions consume and produce
   flags, @cfallin added a way to pass around variants of
   `ConsumesFlags` and `ProducesFlags`--these changes affect all
   backends
 - then, to lower the `fcmp + select` CLIF, this change adds several
   `cmove*_from_values` helpers that perform all of the awkward
   conversions between `Value`, `ValueReg`, `Reg`, and `Gpr/Xmm`; one
   upside is that now these lowerings have much-improved documentation
   explaining why the various `FloatCC` and `CC` choices are made the
   the way they are.

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -1461,37 +1461,42 @@
  (writable_reg_to_reg dst)))
 
 ;; Helper for emitting `adds` instructions.
-(decl add_with_flags (Type Reg Reg) ProducesFlags)
-(rule (add_with_flags ty src1 src2)
+(decl add_with_flags_paired (Type Reg Reg) ProducesFlags)
+(rule (add_with_flags_paired ty src1 src2)
  (let ((dst WritableReg (temp_writable_reg $I64)))
- (ProducesFlags.ProducesFlags (MInst.AluRRR (ALUOp.AddS) (operand_size ty) dst src1 src2)
- (writable_reg_to_reg dst))))
+ (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
+ (MInst.AluRRR (ALUOp.AddS) (operand_size ty) dst src1 src2)
+ (writable_reg_to_reg dst))))
 
 ;; Helper for emitting `adc` instructions.
-(decl adc (Type Reg Reg) ConsumesFlags)
-(rule (adc ty src1 src2)
+(decl adc_paired (Type Reg Reg) ConsumesFlags)
+(rule (adc_paired ty src1 src2)
  (let ((dst WritableReg (temp_writable_reg $I64)))
- (ConsumesFlags.ConsumesFlags (MInst.AluRRR (ALUOp.Adc) (operand_size ty) dst src1 src2)
- (writable_reg_to_reg dst))))
+ (ConsumesFlags.ConsumesFlagsReturnsResultWithProducer
+ (MInst.AluRRR (ALUOp.Adc) (operand_size ty) dst src1 src2)
+ (writable_reg_to_reg dst))))
 
 ;; Helper for emitting `subs` instructions.
-(decl sub_with_flags (Type Reg Reg) ProducesFlags)
-(rule (sub_with_flags ty src1 src2)
+(decl sub_with_flags_paired (Type Reg Reg) ProducesFlags)
+(rule (sub_with_flags_paired ty src1 src2)
  (let ((dst WritableReg (temp_writable_reg $I64)))
- (ProducesFlags.ProducesFlags (MInst.AluRRR (ALUOp.SubS) (operand_size ty) dst src1 src2)
- (writable_reg_to_reg dst))))
+ (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
+ (MInst.AluRRR (ALUOp.SubS) (operand_size ty) dst src1 src2)
+ (writable_reg_to_reg dst))))
 
 (decl cmp64_imm (Reg Imm12) ProducesFlags)
 (rule (cmp64_imm src1 src2)
- (ProducesFlags.ProducesFlags (MInst.AluRRImm12 (ALUOp.SubS) (OperandSize.Size64) (writable_zero_reg) src1 src2)
- (zero_reg)))
+ (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
+ (MInst.AluRRImm12 (ALUOp.SubS) (OperandSize.Size64) (writable_zero_reg)
+ src1 src2) (zero_reg)))
 
 ;; Helper for emitting `sbc` instructions.
-(decl sbc (Type Reg Reg) ConsumesFlags)
-(rule (sbc ty src1 src2)
+(decl sbc_paired (Type Reg Reg) ConsumesFlags)
+(rule (sbc_paired ty src1 src2)
  (let ((dst WritableReg (temp_writable_reg $I64)))
- (ConsumesFlags.ConsumesFlags (MInst.AluRRR (ALUOp.Sbc) (operand_size ty) dst src1 src2)
- (writable_reg_to_reg dst))))
+ (ConsumesFlags.ConsumesFlagsReturnsResultWithProducer
+ (MInst.AluRRR (ALUOp.Sbc) (operand_size ty) dst src1 src2)
+ (writable_reg_to_reg dst))))
 
 ;; Helper for emitting `MInst.VecMisc` instructions.
 (decl vec_misc (VecMisc2 Reg VectorSize) Reg)
@@ -1581,12 +1586,12 @@
 ;; which must be paired with `with_flags*` helpers.
 (decl tst_imm (Type Reg ImmLogic) ProducesFlags)
 (rule (tst_imm ty reg imm)
- (ProducesFlags.ProducesFlags (MInst.AluRRImmLogic (ALUOp.AndS)
-  (operand_size ty)
-  (writable_zero_reg)
-  reg
-  imm)
-  (invalid_reg)))
+ (ProducesFlags.ProducesFlagsSideEffect
+ (MInst.AluRRImmLogic (ALUOp.AndS)
+ (operand_size ty)
+ (writable_zero_reg)
+ reg
+ imm)))
 
 ;; Helper for generating a `CSel` instruction.
 ;;
@@ -1596,8 +1601,9 @@
 (decl csel (Cond Reg Reg) ConsumesFlags)
 (rule (csel cond if_true if_false)
  (let ((dst WritableReg (temp_writable_reg $I64)))
- (ConsumesFlags.ConsumesFlags (MInst.CSel dst cond if_true if_false)
- (writable_reg_to_reg dst))))
+ (ConsumesFlags.ConsumesFlagsReturnsReg
+ (MInst.CSel dst cond if_true if_false)
+ (writable_reg_to_reg dst))))
 
 ;; Helpers for generating `add` instructions.
 

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -91,8 +91,8 @@
  ;; the actual addition is `adds` followed by `adc` which comprises the
  ;; low/high bits of the result
  (with_flags
- (add_with_flags $I64 x_lo y_lo)
- (adc $I64 x_hi y_hi))))
+ (add_with_flags_paired $I64 x_lo y_lo)
+ (adc_paired $I64 x_hi y_hi))))
 
 ;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -142,8 +142,8 @@
  ;; the actual subtraction is `subs` followed by `sbc` which comprises
  ;; the low/high bits of the result
  (with_flags
- (sub_with_flags $I64 x_lo y_lo)
- (sbc $I64 x_hi y_hi))))
+ (sub_with_flags_paired $I64 x_lo y_lo)
+ (sbc_paired $I64 x_hi y_hi))))
 
 ;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -708,10 +708,11 @@
  inv_amt))
  (maybe_hi Reg (orr $I64 hi_lshift lo_rshift))
  )
- (with_flags_2
- (tst_imm $I64 amt (u64_into_imm_logic $I64 64))
+ (with_flags
+ (tst_imm $I64 amt (u64_into_imm_logic $I64 64))
+ (consumes_flags_concat
  (csel (Cond.Ne) (zero_reg) lo_lshift)
- (csel (Cond.Ne) lo_lshift maybe_hi))))
+ (csel (Cond.Ne) lo_lshift maybe_hi)))))
 
 ;; Shift for vector types.
 (rule (lower (has_type (vec128 ty) (ishl x y)))
@@ -805,10 +806,11 @@
  inv_amt))
  (maybe_lo Reg (orr $I64 lo_rshift hi_lshift))
  )
- (with_flags_2
- (tst_imm $I64 amt (u64_into_imm_logic $I64 64))
+ (with_flags
+ (tst_imm $I64 amt (u64_into_imm_logic $I64 64))
+ (consumes_flags_concat
  (csel (Cond.Ne) hi_rshift maybe_lo)
- (csel (Cond.Ne) (zero_reg) hi_rshift))))
+ (csel (Cond.Ne) (zero_reg) hi_rshift)))))
 
 ;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -858,10 +860,11 @@
  (hi_sign Reg (asr_imm $I64 src_hi (imm_shift_from_u8 63)))
  (maybe_lo Reg (orr $I64 lo_rshift hi_lshift))
  )
- (with_flags_2
- (tst_imm $I64 amt (u64_into_imm_logic $I64 64))
+ (with_flags
+ (tst_imm $I64 amt (u64_into_imm_logic $I64 64))
+ (consumes_flags_concat
  (csel (Cond.Ne) hi_rshift maybe_lo)
- (csel (Cond.Ne) hi_sign hi_rshift))))
+ (csel (Cond.Ne) hi_sign hi_rshift)))))
 
 ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1123,9 +1126,9 @@
  (sign_eq_eon Reg (eon $I64 hi lo))
  (sign_eq Reg (lsr_imm $I64 sign_eq_eon (imm_shift_from_u8 63)))
  (lo_sign_bits Reg (madd64 lo_cls sign_eq sign_eq))
- (maybe_lo Reg (with_flags_1
- (cmp64_imm hi_cls (u8_into_imm12 63))
- (csel (Cond.Eq) lo_sign_bits (zero_reg))
+ (maybe_lo Reg (with_flags_reg
+  (cmp64_imm hi_cls (u8_into_imm12 63))
+  (csel (Cond.Eq) lo_sign_bits (zero_reg))
  ))
  )
  (value_regs (add $I64 maybe_lo hi_cls) (imm $I64 0))))

cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 9ea75a6f790b5c03
-src/prelude.isle 73285cd431346d53
-src/isa/aarch64/inst.isle 4c176462894836e5
-src/isa/aarch64/lower.isle aff657984bf30686
+src/prelude.isle 980b300b3ec3e338
+src/isa/aarch64/inst.isle be5744f443a5e400
+src/isa/aarch64/lower.isle 534c135b5f535f33