diff --git a/Cargo.lock b/Cargo.lock index d493499286b2..d87df3e5d1a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -547,7 +547,6 @@ dependencies = [ "miette", "regalloc", "serde", - "sha2", "smallvec", "souper-ir", "target-lexicon", diff --git a/RELEASES.md b/RELEASES.md index 2a4f7600d0f9..1ad2da8ab92a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -2,6 +2,25 @@ -------------------------------------------------------------------------------- +## 0.32.1 + +Released 2021-12-17. + +### Fixed + +* Cranelift: remove recently-added build dependency on `sha2` to allow usage in + some dependency-sensitive environments, by computing ISLE manifest hashes + with a different hash function. + [#3619](https://github.com/bytecodealliance/wasmtime/pull/3619) + +* Cranelift: fixed 8- and 16-bit behavior of popcount (bit population count) + instruction. Does not affect Wasm frontend. + [#3617](https://github.com/bytecodealliance/wasmtime/pull/3617) + +* Cranelift: fixed miscompilation of 8- and 16-bit bit-rotate instructions. + Does not affect Wasm frontend. + [#3610](https://github.com/bytecodealliance/wasmtime/pull/3610) + ## 0.32.0 Released 2021-12-13. diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 77d34bbfb9d5..eee830039eb4 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -37,7 +37,6 @@ criterion = "0.3" cranelift-codegen-meta = { path = "meta", version = "0.79.0" } cranelift-isle = { path = "../isle/isle", version = "=0.79.0", optional = true } miette = { version = "3", features = ["fancy"], optional = true } -sha2 = "0.9.8" [features] default = ["std", "unwind"] diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs index d900b92fa449..8eac58e8d9b2 100644 --- a/cranelift/codegen/build.rs +++ b/cranelift/codegen/build.rs @@ -16,7 +16,6 @@ use cranelift_codegen_meta as meta; -use sha2::{Digest, Sha512}; use std::env; use std::io::Read; use std::process; @@ -163,7 +162,28 @@ impl IsleCompilation { /// `.manifest` and use it to verify that a /// rebuild was done if necessary. fn compute_manifest(&self) -> Result> { + // We use the deprecated SipHasher from std::hash in order to verify + // that ISLE sources haven't changed since the generated source was + // last regenerated. + // + // We use this instead of a stronger and more usual content hash, like + // SHA-{160,256,512}, because it's built into the standard library and + // we don't want to pull in a separate crate. We try to keep Cranelift + // crate dependencies as intentionally small as possible. In fact, we + // used to use the `sha2` crate for SHA-512 and this turns out to be + // undesirable for downstream consumers (see #3609). + // + // Why not the recommended replacement + // `std::collections::hash_map::DefaultHasher`? Because we need the + // hash to be deterministic, both between runs (i.e., not seeded with + // random state) and across Rust versions. + // + // If `SipHasher` is ever actually removed from `std`, we'll need to + // find a new option, either a very small crate or something else + // that's built-in. + #![allow(deprecated)] use std::fmt::Write; + use std::hash::{Hasher, SipHasher}; let mut manifest = String::new(); @@ -176,11 +196,11 @@ impl IsleCompilation { // to `\r\n`; canonicalize the source that we hash to // Unix-style (`\n`) so hashes will match. let content = content.replace("\r\n", "\n"); - // One line in the manifest: . - let mut hasher = Sha512::default(); - hasher.update(content.as_bytes()); + // One line in the manifest: . + let mut hasher = SipHasher::new_with_keys(0, 0); // fixed keys for determinism + hasher.write(content.as_bytes()); let filename = format!("{}", filename.display()).replace("\\", "/"); - writeln!(&mut manifest, "{} {:x}", filename, hasher.finalize())?; + writeln!(&mut manifest, "{} {:x}", filename, hasher.finish())?; } Ok(manifest) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index f832d97efa23..b60de5b25a9b 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ -src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602 -src/prelude.isle 9bd1fcb6a3604a24cf2e05e6b7eb04dcb3b9dc8fa9a2f1c8f29c25b6e3bf7f679b3b1b72dff87501497b72bc30fc92fd755b898db7e03f380235fae931b6a74b -src/isa/aarch64/inst.isle 6e042ec14166fceae4b7133f681fdf604e20a2997e1d60f797e40acd683ccb34e33376189f6b7ed2f5eb441dc61d592cad2592256dfea51296330752181b9403 -src/isa/aarch64/lower.isle 64a725771537f69c445f44c728e04bffd8a715d6a4d87a5a2bf2e89714ee290b7497c5ca8b335bdddd775f6734be03318ff9aa67e2e4068949ebae06b0902b3f +src/clif.isle f176ef3bba99365 +src/prelude.isle 53cfd4f9ccf7c050 +src/isa/aarch64/inst.isle 18c4784ff8c6f057 +src/isa/aarch64/lower.isle 85f6b03feadeb795 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 7bd58b5c39d6..b8df8413a7f0 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -34,6 +34,8 @@ pub trait Context { fn ty_bits_u16(&mut self, arg0: Type) -> u16; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; + fn ty_32_or_64(&mut self, arg0: Type) -> Option; + fn ty_8_or_16(&mut self, arg0: Type) -> Option; fn vec128(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; @@ -48,6 +50,7 @@ pub trait Context { fn first_result(&mut self, arg0: Inst) -> Option; fn inst_data(&mut self, arg0: Inst) -> InstructionData; fn value_type(&mut self, arg0: Value) -> Type; + fn ty_bits_mask(&mut self, arg0: Type) -> u64; fn multi_lane(&mut self, arg0: Type) -> Option<(u8, u16)>; fn def_inst(&mut self, arg0: Value) -> Option; fn move_wide_const_from_u64(&mut self, arg0: u64) -> Option; @@ -66,13 +69,13 @@ pub trait Context { fn load_constant64_full(&mut self, arg0: u64) -> Reg; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 242. +/// Internal type ProducesFlags: defined at src/prelude.isle line 255. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 245. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 258. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -986,7 +989,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 255. + // Rule at src/prelude.isle line 268. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1014,7 +1017,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 263. + // Rule at src/prelude.isle line 276. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1048,7 +1051,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 273. + // Rule at src/prelude.isle line 286. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 674cd3dd1945..33080ae6485c 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -77,9 +77,13 @@ Size32 Size64)) -;; Get the `OperandSize` for a given `Type`. -(decl operand_size_of_type (Type) OperandSize) -(extern constructor operand_size_of_type operand_size_of_type) +;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits. +(decl operand_size_of_type_32_64 (Type) OperandSize) +(extern constructor operand_size_of_type_32_64 operand_size_of_type_32_64) + +;; Get the true `OperandSize` for a given `Type`, with no rounding. +(decl raw_operand_size_of_type (Type) OperandSize) +(extern constructor raw_operand_size_of_type raw_operand_size_of_type) ;; Get the bit width of an `OperandSize`. (decl operand_size_bits (OperandSize) u16) @@ -379,6 +383,10 @@ (decl imm8_from_value (Imm8Reg) Value) (extern extractor imm8_from_value imm8_from_value) +;; Mask an `Imm8Reg.Imm8`. +(decl mask_imm8_const (Imm8Reg u64) Imm8Reg) +(extern constructor mask_imm8_const mask_imm8_const) + ;; Extract a constant `RegMemImm.Imm` from a value operand. (decl simm32_from_value (RegMemImm) Value) (extern extractor simm32_from_value simm32_from_value) @@ -426,7 +434,7 @@ (let ((from_bits u16 (ty_bits_u16 from_ty)) ;; Use `operand_size_of_type` so that the we clamp the output to 32- ;; or 64-bit width types. - (to_bits u16 (operand_size_bits (operand_size_of_type to_ty)))) + (to_bits u16 (operand_size_bits (operand_size_of_type_32_64 to_ty)))) (extend kind to_ty (ext_mode from_bits to_bits) @@ -503,7 +511,7 @@ (decl alu_rmi_r (Type AluRmiROpcode Reg RegMemImm) Reg) (rule (alu_rmi_r ty opcode src1 src2) (let ((dst WritableReg (temp_writable_reg ty)) - (size OperandSize (operand_size_of_type ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.AluRmiR size opcode src1 src2 dst)))) (writable_reg_to_reg dst))) @@ -519,7 +527,7 @@ (decl add_with_flags (Type Reg RegMemImm) ProducesFlags) (rule (add_with_flags ty src1 src2) (let ((dst WritableReg (temp_writable_reg ty))) - (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type ty) + (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty) (AluRmiROpcode.Add) src1 src2 @@ -530,7 +538,7 @@ (decl adc (Type Reg RegMemImm) ConsumesFlags) (rule (adc ty src1 src2) (let ((dst WritableReg (temp_writable_reg ty))) - (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type ty) + (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty) (AluRmiROpcode.Adc) src1 src2 @@ -549,7 +557,7 @@ (decl sub_with_flags (Type Reg RegMemImm) ProducesFlags) (rule (sub_with_flags ty src1 src2) (let ((dst WritableReg (temp_writable_reg ty))) - (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type ty) + (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty) (AluRmiROpcode.Sub) src1 src2 @@ -560,7 +568,7 @@ (decl sbb (Type Reg RegMemImm) ConsumesFlags) (rule (sbb ty src1 src2) (let ((dst WritableReg (temp_writable_reg ty))) - (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type ty) + (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty) (AluRmiROpcode.Sbb) src1 src2 @@ -608,7 +616,7 @@ ;; Integer immediates. (rule (imm ty simm64) (let ((dst WritableReg (temp_writable_reg ty)) - (size OperandSize (operand_size_of_type ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Imm size simm64 dst)))) (writable_reg_to_reg dst))) @@ -634,7 +642,7 @@ (rule (imm ty 0) (let ((wr WritableReg (temp_writable_reg ty)) (r Reg (writable_reg_to_reg wr)) - (size OperandSize (operand_size_of_type ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.AluRmiR size (AluRmiROpcode.Xor) r @@ -681,7 +689,9 @@ (decl shift_r (Type ShiftKind Reg Imm8Reg) Reg) (rule (shift_r ty kind src1 src2) (let ((dst WritableReg (temp_writable_reg ty)) - (size OperandSize (operand_size_of_type ty)) + ;; Use actual 8/16-bit instructions when appropriate: we + ;; rely on their shift-amount-masking semantics. + (size OperandSize (raw_operand_size_of_type ty)) (_ Unit (emit (MInst.ShiftR size kind src1 src2 dst)))) (writable_reg_to_reg dst))) @@ -729,7 +739,7 @@ (decl cmove (Type CC RegMem Reg) ConsumesFlags) (rule (cmove ty cc consequent alternative) (let ((dst WritableReg (temp_writable_reg ty)) - (size OperandSize (operand_size_of_type ty))) + (size OperandSize (operand_size_of_type_32_64 ty))) (ConsumesFlags.ConsumesFlags (MInst.Cmove size cc consequent alternative dst) (writable_reg_to_reg dst)))) @@ -1155,7 +1165,7 @@ (rule (mul_hi ty signed src1 src2) (let ((dst_lo WritableReg (temp_writable_reg ty)) (dst_hi WritableReg (temp_writable_reg ty)) - (size OperandSize (operand_size_of_type ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.MulHi size signed src1 @@ -1224,6 +1234,6 @@ (decl not (Type Reg) Reg) (rule (not ty src) (let ((dst WritableReg (temp_writable_reg ty)) - (size OperandSize (operand_size_of_type ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Not size src dst)))) (writable_reg_to_reg dst))) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index b8b22f2209d3..06d064f7d7ea 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -633,15 +633,26 @@ ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; `i64` and smaller. +;; `i16` and `i8`: we need to extend the shift amount, or mask the +;; constant. + +(rule (lower (has_type (ty_8_or_16 ty) (rotl src amt))) + (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero)))) + (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + +(rule (lower (has_type (ty_8_or_16 ty) (rotl src (imm8_from_value amt)))) + (value_reg (m_rotl ty (put_in_reg src) (mask_imm8_const amt (ty_bits_mask ty))))) + +;; `i64` and `i32`: we can rely on x86's rotate-amount masking since +;; we operate on the whole register. -(rule (lower (has_type (fits_in_64 ty) (rotl src amt))) +(rule (lower (has_type (ty_32_or_64 ty) (rotl src amt))) ;; NB: Only the low bits of `amt` matter since we logically mask the ;; shift amount to the value's bit width. (let ((amt_ Reg (lo_reg amt))) (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) -(rule (lower (has_type (fits_in_64 ty) (rotl src (imm8_from_value amt)))) +(rule (lower (has_type (ty_32_or_64 ty) (rotl src (imm8_from_value amt)))) (value_reg (m_rotl ty (put_in_reg src) amt))) ;; `i128`. diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index f75a9511bd24..d0d0debfdf83 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2236,14 +2236,25 @@ fn lower_insn_to_regs>( Opcode::Popcnt => { let ty_tmp = ty.unwrap(); if !ty_tmp.is_vector() { - let (ext_spec, ty) = match ctx.input_ty(insn, 0) { - types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32), - a if a == types::I32 || a == types::I64 || a == types::I128 => (None, a), - _ => unreachable!(), - }; + let ty = ctx.input_ty(insn, 0); if isa_flags.use_popcnt() { match ty { + types::I8 | types::I16 => { + let src = RegMem::reg(extend_input_to_reg( + ctx, + inputs[0], + ExtSpec::ZeroExtendTo32, + )); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + ctx.emit(Inst::unary_rm_r( + OperandSize::from_ty(types::I32), + UnaryRmROpcode::Popcnt, + src, + dst, + )); + return Ok(()); + } types::I32 | types::I64 => { let src = input_to_reg_mem(ctx, inputs[0]); let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); @@ -2299,6 +2310,12 @@ fn lower_insn_to_regs>( } } + let (ext_spec, ty) = match ty { + types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32), + a if a == types::I32 || a == types::I64 || a == types::I128 => (None, a), + _ => unreachable!(), + }; + let (srcs, ty): (SmallVec<[RegMem; 2]>, Type) = if let Some(ext_spec) = ext_spec { ( smallvec![RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec))], diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 380ab29a8a27..af69ae7ed76d 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -111,7 +111,7 @@ where isle_prelude_methods!(); #[inline] - fn operand_size_of_type(&mut self, ty: Type) -> OperandSize { + fn operand_size_of_type_32_64(&mut self, ty: Type) -> OperandSize { if ty.bits() == 64 { OperandSize::Size64 } else { @@ -119,6 +119,11 @@ where } } + #[inline] + fn raw_operand_size_of_type(&mut self, ty: Type) -> OperandSize { + OperandSize::from_ty(ty) + } + fn put_in_reg_mem(&mut self, val: Value) -> RegMem { let inputs = self.lower_ctx.get_value_as_source_or_const(val); @@ -180,6 +185,16 @@ where Some(Imm8Reg::Imm8 { imm }) } + #[inline] + fn mask_imm8_const(&mut self, imm8: &Imm8Reg, mask: u64) -> Imm8Reg { + match imm8 { + &Imm8Reg::Reg { reg } => Imm8Reg::Reg { reg }, + &Imm8Reg::Imm8 { imm } => Imm8Reg::Imm8 { + imm: imm & (mask as u8), + }, + } + } + #[inline] fn simm32_from_value(&mut self, val: Value) -> Option { let inst = self.lower_ctx.dfg().value_def(val).inst()?; diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 570c14b1af9a..42953806d322 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ -src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602 -src/prelude.isle 9bd1fcb6a3604a24cf2e05e6b7eb04dcb3b9dc8fa9a2f1c8f29c25b6e3bf7f679b3b1b72dff87501497b72bc30fc92fd755b898db7e03f380235fae931b6a74b -src/isa/x64/inst.isle b151120df3c356ac697122a8557becd8857eb725851506e844edeb85d831d461322a96d280ad84f9a23518e1e4efb607aebc0e249004148675e4cc19e89f0655 -src/isa/x64/lower.isle c9b408df0a089fb4f207838973ac775b0f9b56c86f056867c28e6bae317873d3844f74f713f9acd6fed98d3d11a2f9d19d392fe5049169dad33b1fc703b9b766 +src/clif.isle f176ef3bba99365 +src/prelude.isle 53cfd4f9ccf7c050 +src/isa/x64/inst.isle fb5d3ac8e68c46d2 +src/isa/x64/lower.isle d39e01add89178d5 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 30e1f2d4902c..f93067ef130f 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -34,6 +34,8 @@ pub trait Context { fn ty_bits_u16(&mut self, arg0: Type) -> u16; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; + fn ty_32_or_64(&mut self, arg0: Type) -> Option; + fn ty_8_or_16(&mut self, arg0: Type) -> Option; fn vec128(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; @@ -48,9 +50,11 @@ pub trait Context { fn first_result(&mut self, arg0: Inst) -> Option; fn inst_data(&mut self, arg0: Inst) -> InstructionData; fn value_type(&mut self, arg0: Value) -> Type; + fn ty_bits_mask(&mut self, arg0: Type) -> u64; fn multi_lane(&mut self, arg0: Type) -> Option<(u8, u16)>; fn def_inst(&mut self, arg0: Value) -> Option; - fn operand_size_of_type(&mut self, arg0: Type) -> OperandSize; + fn operand_size_of_type_32_64(&mut self, arg0: Type) -> OperandSize; + fn raw_operand_size_of_type(&mut self, arg0: Type) -> OperandSize; fn put_in_reg_mem(&mut self, arg0: Value) -> RegMem; fn encode_fcmp_imm(&mut self, arg0: &FcmpImm) -> u8; fn xmm0(&mut self) -> WritableReg; @@ -58,6 +62,7 @@ pub trait Context { fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512f_enabled(&mut self, arg0: Type) -> Option<()>; fn imm8_from_value(&mut self, arg0: Value) -> Option; + fn mask_imm8_const(&mut self, arg0: &Imm8Reg, arg1: u64) -> Imm8Reg; fn simm32_from_value(&mut self, arg0: Value) -> Option; fn simm32_from_imm64(&mut self, arg0: Imm64) -> Option; fn sinkable_load(&mut self, arg0: Value) -> Option; @@ -68,19 +73,19 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 242. +/// Internal type ProducesFlags: defined at src/prelude.isle line 255. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 245. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 258. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 408. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 416. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ExtendKind { Sign, @@ -124,7 +129,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 255. + // Rule at src/prelude.isle line 268. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -152,7 +157,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 263. + // Rule at src/prelude.isle line 276. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -186,7 +191,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 273. + // Rule at src/prelude.isle line 286. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); @@ -203,22 +208,22 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize let pattern0_0 = arg0; match pattern0_0 { &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 86. + // Rule at src/isa/x64/inst.isle line 90. let expr0_0: u16 = 8; return Some(expr0_0); } &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 87. + // Rule at src/isa/x64/inst.isle line 91. let expr0_0: u16 = 16; return Some(expr0_0); } &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 88. + // Rule at src/isa/x64/inst.isle line 92. let expr0_0: u16 = 32; return Some(expr0_0); } &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 89. + // Rule at src/isa/x64/inst.isle line 93. let expr0_0: u16 = 64; return Some(expr0_0); } @@ -239,14 +244,14 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 420. + // Rule at src/isa/x64/inst.isle line 428. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 423. + // Rule at src/isa/x64/inst.isle line 431. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); - let expr1_0 = C::operand_size_of_type(ctx, pattern2_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; let expr3_0 = C::ext_mode(ctx, expr0_0, expr2_0); let expr4_0 = C::put_in_reg_mem(ctx, pattern0_0); @@ -268,7 +273,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 443. + // Rule at src/isa/x64/inst.isle line 451. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -276,7 +281,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 439. + // Rule at src/isa/x64/inst.isle line 447. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -289,17 +294,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 450. + // Rule at src/isa/x64/inst.isle line 458. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 451. + // Rule at src/isa/x64/inst.isle line 459. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 452. + // Rule at src/isa/x64/inst.isle line 460. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -316,7 +321,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 456. + // Rule at src/isa/x64/inst.isle line 464. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -326,40 +331,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 465. + // Rule at src/isa/x64/inst.isle line 473. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 466. + // Rule at src/isa/x64/inst.isle line 474. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 461. + // Rule at src/isa/x64/inst.isle line 469. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 462. + // Rule at src/isa/x64/inst.isle line 470. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 463. + // Rule at src/isa/x64/inst.isle line 471. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 464. + // Rule at src/isa/x64/inst.isle line 472. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -371,7 +376,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 480. + // Rule at src/isa/x64/inst.isle line 488. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0: Type = I32X4; @@ -399,9 +404,9 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 504. + // Rule at src/isa/x64/inst.isle line 512. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { size: expr1_0, op: pattern1_0.clone(), @@ -424,7 +429,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 512. + // Rule at src/isa/x64/inst.isle line 520. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -440,9 +445,9 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 520. + // Rule at src/isa/x64/inst.isle line 528. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; let expr3_0 = MInst::AluRmiR { size: expr1_0, @@ -469,9 +474,9 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 531. + // Rule at src/isa/x64/inst.isle line 539. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; let expr3_0 = MInst::AluRmiR { size: expr1_0, @@ -498,7 +503,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 542. + // Rule at src/isa/x64/inst.isle line 550. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -514,9 +519,9 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 550. + // Rule at src/isa/x64/inst.isle line 558. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; let expr3_0 = MInst::AluRmiR { size: expr1_0, @@ -543,9 +548,9 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 561. + // Rule at src/isa/x64/inst.isle line 569. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; let expr3_0 = MInst::AluRmiR { size: expr1_0, @@ -572,7 +577,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 572. + // Rule at src/isa/x64/inst.isle line 580. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -588,7 +593,7 @@ pub fn constructor_m_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 583. + // Rule at src/isa/x64/inst.isle line 591. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -604,7 +609,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 591. + // Rule at src/isa/x64/inst.isle line 599. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -620,7 +625,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 599. + // Rule at src/isa/x64/inst.isle line 607. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -632,7 +637,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 628. + // Rule at src/isa/x64/inst.isle line 636. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -649,7 +654,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 657. + // Rule at src/isa/x64/inst.isle line 665. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -664,7 +669,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 616. + // Rule at src/isa/x64/inst.isle line 624. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Movd; let expr2_0: Type = I32; @@ -677,7 +682,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 669. + // Rule at src/isa/x64/inst.isle line 677. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -692,7 +697,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 620. + // Rule at src/isa/x64/inst.isle line 628. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Movq; let expr2_0: Type = I64; @@ -705,7 +710,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 647. + // Rule at src/isa/x64/inst.isle line 655. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -722,10 +727,10 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option } let pattern1_0 = arg1; if pattern1_0 == 0 { - // Rule at src/isa/x64/inst.isle line 634. + // Rule at src/isa/x64/inst.isle line 642. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr3_0 = AluRmiROpcode::Xor; let expr4_0 = RegMemImm::Reg { reg: expr1_0 }; let expr5_0 = MInst::AluRmiR { @@ -738,9 +743,9 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 609. + // Rule at src/isa/x64/inst.isle line 617. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Imm { dst_size: expr1_0, simm64: pattern1_0, @@ -763,9 +768,9 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 682. + // Rule at src/isa/x64/inst.isle line 690. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { size: expr1_0, kind: pattern1_0.clone(), @@ -788,7 +793,7 @@ pub fn constructor_m_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 691. + // Rule at src/isa/x64/inst.isle line 701. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -804,7 +809,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 696. + // Rule at src/isa/x64/inst.isle line 706. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -820,7 +825,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 701. + // Rule at src/isa/x64/inst.isle line 711. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -836,7 +841,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 706. + // Rule at src/isa/x64/inst.isle line 716. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -854,7 +859,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 711. + // Rule at src/isa/x64/inst.isle line 721. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -879,7 +884,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 720. + // Rule at src/isa/x64/inst.isle line 730. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -895,7 +900,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 725. + // Rule at src/isa/x64/inst.isle line 735. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -913,9 +918,9 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 730. + // Rule at src/isa/x64/inst.isle line 740. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { size: expr1_0, cc: pattern1_0.clone(), @@ -941,7 +946,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 738. + // Rule at src/isa/x64/inst.isle line 748. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -963,7 +968,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 745. + // Rule at src/isa/x64/inst.isle line 755. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -987,7 +992,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 752. + // Rule at src/isa/x64/inst.isle line 762. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -1004,7 +1009,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 759. + // Rule at src/isa/x64/inst.isle line 769. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1015,7 +1020,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 764. + // Rule at src/isa/x64/inst.isle line 774. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1026,7 +1031,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 769. + // Rule at src/isa/x64/inst.isle line 779. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1037,7 +1042,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 774. + // Rule at src/isa/x64/inst.isle line 784. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1048,7 +1053,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 779. + // Rule at src/isa/x64/inst.isle line 789. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1059,7 +1064,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 784. + // Rule at src/isa/x64/inst.isle line 794. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1070,7 +1075,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 789. + // Rule at src/isa/x64/inst.isle line 799. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1081,7 +1086,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 794. + // Rule at src/isa/x64/inst.isle line 804. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1092,7 +1097,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 799. + // Rule at src/isa/x64/inst.isle line 809. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1103,7 +1108,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 804. + // Rule at src/isa/x64/inst.isle line 814. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1114,7 +1119,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 809. + // Rule at src/isa/x64/inst.isle line 819. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1125,7 +1130,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 814. + // Rule at src/isa/x64/inst.isle line 824. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1136,7 +1141,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 819. + // Rule at src/isa/x64/inst.isle line 829. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1147,7 +1152,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 824. + // Rule at src/isa/x64/inst.isle line 834. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1158,7 +1163,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 829. + // Rule at src/isa/x64/inst.isle line 839. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1169,7 +1174,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 834. + // Rule at src/isa/x64/inst.isle line 844. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1180,7 +1185,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 839. + // Rule at src/isa/x64/inst.isle line 849. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1191,7 +1196,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 844. + // Rule at src/isa/x64/inst.isle line 854. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1202,7 +1207,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 849. + // Rule at src/isa/x64/inst.isle line 859. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1213,7 +1218,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 854. + // Rule at src/isa/x64/inst.isle line 864. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1224,7 +1229,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 859. + // Rule at src/isa/x64/inst.isle line 869. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1235,7 +1240,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 864. + // Rule at src/isa/x64/inst.isle line 874. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1246,7 +1251,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 869. + // Rule at src/isa/x64/inst.isle line 879. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1257,7 +1262,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 874. + // Rule at src/isa/x64/inst.isle line 884. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1268,7 +1273,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 879. + // Rule at src/isa/x64/inst.isle line 889. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1279,7 +1284,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 884. + // Rule at src/isa/x64/inst.isle line 894. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1290,7 +1295,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 889. + // Rule at src/isa/x64/inst.isle line 899. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1301,7 +1306,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 894. + // Rule at src/isa/x64/inst.isle line 904. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1312,7 +1317,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 899. + // Rule at src/isa/x64/inst.isle line 909. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1323,7 +1328,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 904. + // Rule at src/isa/x64/inst.isle line 914. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1334,7 +1339,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 909. + // Rule at src/isa/x64/inst.isle line 919. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1345,7 +1350,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 914. + // Rule at src/isa/x64/inst.isle line 924. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1356,7 +1361,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 919. + // Rule at src/isa/x64/inst.isle line 929. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1367,7 +1372,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 924. + // Rule at src/isa/x64/inst.isle line 934. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1378,7 +1383,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 929. + // Rule at src/isa/x64/inst.isle line 939. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1389,7 +1394,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 934. + // Rule at src/isa/x64/inst.isle line 944. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1400,7 +1405,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 939. + // Rule at src/isa/x64/inst.isle line 949. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1411,7 +1416,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 944. + // Rule at src/isa/x64/inst.isle line 954. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1428,7 +1433,7 @@ pub fn constructor_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 949. + // Rule at src/isa/x64/inst.isle line 959. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = RegMem::Reg { reg: pattern2_0 }; @@ -1448,7 +1453,7 @@ pub fn constructor_blendvpd( pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 961. + // Rule at src/isa/x64/inst.isle line 971. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1459,7 +1464,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 966. + // Rule at src/isa/x64/inst.isle line 976. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1470,7 +1475,7 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 971. + // Rule at src/isa/x64/inst.isle line 981. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1481,7 +1486,7 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 976. + // Rule at src/isa/x64/inst.isle line 986. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1492,7 +1497,7 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 981. + // Rule at src/isa/x64/inst.isle line 991. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1503,7 +1508,7 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 986. + // Rule at src/isa/x64/inst.isle line 996. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1514,7 +1519,7 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 991. + // Rule at src/isa/x64/inst.isle line 1001. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1525,7 +1530,7 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 996. + // Rule at src/isa/x64/inst.isle line 1006. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1536,7 +1541,7 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1001. + // Rule at src/isa/x64/inst.isle line 1011. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1547,7 +1552,7 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1006. + // Rule at src/isa/x64/inst.isle line 1016. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1558,7 +1563,7 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1011. + // Rule at src/isa/x64/inst.isle line 1021. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1569,7 +1574,7 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1016. + // Rule at src/isa/x64/inst.isle line 1026. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1580,7 +1585,7 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1021. + // Rule at src/isa/x64/inst.isle line 1031. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1591,7 +1596,7 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1026. + // Rule at src/isa/x64/inst.isle line 1036. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1612,7 +1617,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1031. + // Rule at src/isa/x64/inst.isle line 1041. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1640,7 +1645,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1043. + // Rule at src/isa/x64/inst.isle line 1053. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1658,7 +1663,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1052. + // Rule at src/isa/x64/inst.isle line 1062. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1683,7 +1688,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1065. + // Rule at src/isa/x64/inst.isle line 1075. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1699,7 +1704,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1072. + // Rule at src/isa/x64/inst.isle line 1082. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1708,7 +1713,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1077. + // Rule at src/isa/x64/inst.isle line 1087. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1717,7 +1722,7 @@ pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1082. + // Rule at src/isa/x64/inst.isle line 1092. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1726,7 +1731,7 @@ pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsw. pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1087. + // Rule at src/isa/x64/inst.isle line 1097. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1735,7 +1740,7 @@ pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsd. pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1092. + // Rule at src/isa/x64/inst.isle line 1102. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1749,7 +1754,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1097. + // Rule at src/isa/x64/inst.isle line 1107. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmREvex { @@ -1765,7 +1770,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term vpabsq. pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1104. + // Rule at src/isa/x64/inst.isle line 1114. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1781,7 +1786,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1109. + // Rule at src/isa/x64/inst.isle line 1119. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -1799,7 +1804,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1121. + // Rule at src/isa/x64/inst.isle line 1131. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1815,7 +1820,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1128. + // Rule at src/isa/x64/inst.isle line 1138. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -1833,7 +1838,7 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1138. + // Rule at src/isa/x64/inst.isle line 1148. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1843,7 +1848,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1143. + // Rule at src/isa/x64/inst.isle line 1153. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1853,7 +1858,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1148. + // Rule at src/isa/x64/inst.isle line 1158. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1871,10 +1876,10 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1155. + // Rule at src/isa/x64/inst.isle line 1165. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr3_0 = MInst::MulHi { size: expr2_0, signed: pattern1_0, @@ -1900,7 +1905,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1171. + // Rule at src/isa/x64/inst.isle line 1181. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1916,7 +1921,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1176. + // Rule at src/isa/x64/inst.isle line 1186. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -1935,7 +1940,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1189. + // Rule at src/isa/x64/inst.isle line 1199. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -1956,7 +1961,7 @@ pub fn constructor_gpr_to_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1198. + // Rule at src/isa/x64/inst.isle line 1208. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::GprToXmm { op: pattern1_0.clone(), @@ -1979,7 +1984,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1205. + // Rule at src/isa/x64/inst.isle line 1215. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -1997,7 +2002,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1210. + // Rule at src/isa/x64/inst.isle line 1220. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2017,7 +2022,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1215. + // Rule at src/isa/x64/inst.isle line 1225. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2035,7 +2040,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1220. + // Rule at src/isa/x64/inst.isle line 1230. let expr0_0 = SseOpcode::Insertps; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2047,9 +2052,9 @@ pub fn constructor_insertps( pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1225. + // Rule at src/isa/x64/inst.isle line 1235. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { size: expr1_0, src: pattern1_0, @@ -2100,7 +2105,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1028. + // Rule at src/isa/x64/lower.isle line 1039. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2289,7 +2294,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 716. + // Rule at src/isa/x64/lower.isle line 727. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2380,7 +2385,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 649. + // Rule at src/isa/x64/lower.isle line 660. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?; @@ -2420,7 +2425,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1025. + // Rule at src/isa/x64/lower.isle line 1036. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2465,7 +2470,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1106. + // Rule at src/isa/x64/lower.isle line 1117. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; @@ -2475,7 +2480,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1128. + // Rule at src/isa/x64/lower.isle line 1139. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; @@ -2485,7 +2490,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1095. + // Rule at src/isa/x64/lower.isle line 1106. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -2495,7 +2500,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1117. + // Rule at src/isa/x64/lower.isle line 1128. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; @@ -2510,7 +2515,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 971. + // Rule at src/isa/x64/lower.isle line 982. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2531,7 +2536,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1109. + // Rule at src/isa/x64/lower.isle line 1120. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; @@ -2541,7 +2546,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1131. + // Rule at src/isa/x64/lower.isle line 1142. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; @@ -2551,7 +2556,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1098. + // Rule at src/isa/x64/lower.isle line 1109. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -2561,7 +2566,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1120. + // Rule at src/isa/x64/lower.isle line 1131. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; @@ -2576,7 +2581,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 974. + // Rule at src/isa/x64/lower.isle line 985. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2597,7 +2602,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1112. + // Rule at src/isa/x64/lower.isle line 1123. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; @@ -2607,7 +2612,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1134. + // Rule at src/isa/x64/lower.isle line 1145. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; @@ -2617,7 +2622,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1101. + // Rule at src/isa/x64/lower.isle line 1112. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -2627,7 +2632,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1123. + // Rule at src/isa/x64/lower.isle line 1134. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; @@ -2642,7 +2647,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 977. + // Rule at src/isa/x64/lower.isle line 988. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2660,7 +2665,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 960. + // Rule at src/isa/x64/lower.isle line 971. let expr0_0 = C::put_in_reg(ctx, pattern7_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr2_0 = constructor_andnps(ctx, expr0_0, &expr1_0)?; @@ -2720,7 +2725,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Fabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 999. + // Rule at src/isa/x64/lower.isle line 1010. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0: Type = F32X4; let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?; @@ -2767,7 +2772,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 963. + // Rule at src/isa/x64/lower.isle line 974. let expr0_0 = C::put_in_reg(ctx, pattern7_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr2_0 = constructor_andnpd(ctx, expr0_0, &expr1_0)?; @@ -2782,7 +2787,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Fabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1004. + // Rule at src/isa/x64/lower.isle line 1015. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0: Type = F64X2; let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?; @@ -2825,7 +2830,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 659. + // Rule at src/isa/x64/lower.isle line 670. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; @@ -2953,7 +2958,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 663. + // Rule at src/isa/x64/lower.isle line 674. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; @@ -3066,7 +3071,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 966. + // Rule at src/isa/x64/lower.isle line 977. let expr0_0 = C::put_in_reg(ctx, pattern7_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr2_0 = constructor_pandn(ctx, expr0_0, &expr1_0)?; @@ -4004,7 +4009,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1033. + // Rule at src/isa/x64/lower.isle line 1044. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4131,7 +4136,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - let (pattern7_0, pattern7_1) = - C::unpack_value_array_2(ctx, &pattern5_1); - if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) { - // Rule at src/isa/x64/lower.isle line 644. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = - constructor_m_rotl(ctx, pattern3_0, expr0_0, &pattern8_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); - } - // Rule at src/isa/x64/lower.isle line 638. - let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; - let expr1_0 = C::put_in_reg(ctx, pattern7_0); - let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; - let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); - return Some(expr4_0); - } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); @@ -4429,7 +4415,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1012. + // Rule at src/isa/x64/lower.isle line 1023. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -4455,6 +4441,62 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } } + if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + if let &Opcode::Rotl = &pattern5_0 { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 655. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = constructor_m_rotl(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + // Rule at src/isa/x64/lower.isle line 649. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; + let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + } + } + if let Some(pattern3_0) = C::ty_8_or_16(ctx, pattern2_0) { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + if let &Opcode::Rotl = &pattern5_0 { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 643. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::ty_bits_mask(ctx, pattern3_0); + let expr2_0 = C::mask_imm8_const(ctx, &pattern8_0, expr1_0); + let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + // Rule at src/isa/x64/lower.isle line 639. + let expr0_0: Type = I32; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = constructor_extend_to_reg(ctx, pattern7_1, expr0_0, &expr1_0)?; + let expr3_0 = C::put_in_reg(ctx, pattern7_0); + let expr4_0 = Imm8Reg::Reg { reg: expr2_0 }; + let expr5_0 = constructor_m_rotl(ctx, pattern3_0, expr3_0, &expr4_0)?; + let expr6_0 = C::value_reg(ctx, expr5_0); + return Some(expr6_0); + } + } + } } return None; } @@ -4610,7 +4652,7 @@ pub fn constructor_shr_i128( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1018. + // Rule at src/isa/x64/lower.isle line 1029. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -4637,7 +4679,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1049. + // Rule at src/isa/x64/lower.isle line 1060. let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -4645,7 +4687,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1052. + // Rule at src/isa/x64/lower.isle line 1063. let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -4653,7 +4695,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1055. + // Rule at src/isa/x64/lower.isle line 1066. let expr0_0 = OperandSize::Size32; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -4662,7 +4704,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1058. + // Rule at src/isa/x64/lower.isle line 1069. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -4671,7 +4713,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1061. + // Rule at src/isa/x64/lower.isle line 1072. let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; return Some(expr1_0); @@ -4682,7 +4724,7 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1082. + // Rule at src/isa/x64/lower.isle line 1093. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -4690,7 +4732,7 @@ pub fn constructor_vec_insert_lane( } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1083. + // Rule at src/isa/x64/lower.isle line 1094. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4698,7 +4740,7 @@ pub fn constructor_vec_insert_lane( return Some(expr3_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1091. + // Rule at src/isa/x64/lower.isle line 1102. let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index c969ac01064c..96f44deb25bf 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -91,6 +91,11 @@ macro_rules! isle_prelude_methods { ty.bits().try_into().unwrap() } + #[inline] + fn ty_bits_mask(&mut self, ty: Type) -> u64 { + (1 << (self.ty_bits(ty) as u64)) - 1 + } + #[inline] fn ty_bits_u16(&mut self, ty: Type) -> u16 { ty.bits() @@ -114,6 +119,24 @@ macro_rules! isle_prelude_methods { } } + #[inline] + fn ty_32_or_64(&mut self, ty: Type) -> Option { + if ty.bits() == 32 || ty.bits() == 64 { + Some(ty) + } else { + None + } + } + + #[inline] + fn ty_8_or_16(&mut self, ty: Type) -> Option { + if ty.bits() == 8 || ty.bits() == 16 { + Some(ty) + } else { + None + } + } + fn vec128(&mut self, ty: Type) -> Option { if ty.is_vector() && ty.bits() == 128 { Some(ty) diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 8b17e53f4f5a..c3f9c2d65c89 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -156,6 +156,14 @@ (decl fits_in_64 (Type) Type) (extern extractor fits_in_64 fits_in_64) +;; An extractor that maches 32- and 64-bit types only. +(decl ty_32_or_64 (Type) Type) +(extern extractor ty_32_or_64 ty_32_or_64) + +;; An extractor that maches 8- and 16-bit types only. +(decl ty_8_or_16 (Type) Type) +(extern extractor ty_8_or_16 ty_8_or_16) + ;; An extractor that only matches 128-bit vector types. (decl vec128 (Type) Type) (extern extractor vec128 vec128) @@ -226,6 +234,11 @@ (and (result_type ty) inst)) +;; Return a bitmask that will mask off a count to be within `ty`'s +;; bit-width. Used for shifts/rotates. +(decl ty_bits_mask (Type) u64) +(extern constructor ty_bits_mask ty_bits_mask) + ;; Match a multi-lane type, extracting (# bits per lane, # lanes) from the given ;; type. Will only match when there is more than one lane. (decl multi_lane (u8 u16) Type) diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif index 5066404fe0be..55a2f48b4dc1 100644 --- a/cranelift/filetests/filetests/isa/x64/i128.clif +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -888,7 +888,7 @@ block0(v0: i8, v1: i128): ; check: pushq %rbp ; nextln: movq %rsp, %rbp ; nextln: movq %rsi, %rcx -; nextln: shll %cl, %edi +; nextln: shlb %cl, %dil ; nextln: movq %rdi, %rax ; nextln: movq %rbp, %rsp ; nextln: popq %rbp diff --git a/cranelift/filetests/filetests/runtests/popcnt-aarch64.clif b/cranelift/filetests/filetests/runtests/popcnt-aarch64.clif deleted file mode 100644 index 99c2b2eca849..000000000000 --- a/cranelift/filetests/filetests/runtests/popcnt-aarch64.clif +++ /dev/null @@ -1,50 +0,0 @@ -test interpret -test run -target aarch64 - -function %popcnt_i8(i8) -> i8 { -block0(v0: i8): - v1 = popcnt v0 - return v1 -} -; run: %popcnt_i8(1) == 1 -; run: %popcnt_i8(0x40) == 1 -; run: %popcnt_i8(-1) == 8 -; run: %popcnt_i8(0) == 0 - -function %popcnt_i16(i16) -> i16 { -block0(v0: i16): - v1 = popcnt v0 - return v1 -} -; run: %popcnt_i16(1) == 1 -; run: %popcnt_i16(0x4000) == 1 -; run: %popcnt_i16(-1) == 16 -; run: %popcnt_i16(0) == 0 - -function %popcnt_i32(i32) -> i32 { -block0(v0: i32): - v1 = popcnt v0 - return v1 -} -; run: %popcnt_i32(1) == 1 -; run: %popcnt_i32(0x40000000) == 1 -; run: %popcnt_i32(-1) == 32 -; run: %popcnt_i32(0) == 0 - -function %popcnt_i64(i64) -> i64 { -block0(v0: i64): - v1 = popcnt v0 - return v1 -} -; run: %popcnt_i64(1) == 1 -; run: %popcnt_i64(0x4000000000000000) == 1 -; run: %popcnt_i64(-1) == 64 -; run: %popcnt_i64(0) == 0 - -function %popcnt_i8x16(i8x16) -> i8x16 { -block0(v0: i8x16): - v1 = popcnt v0 - return v1 -} -; run: %popcnt_i8x16([1 1 1 1 0x40 0x40 0x40 0x40 0xff 0xff 0xff 0xff 0 0 0 0]) == [1 1 1 1 1 1 1 1 8 8 8 8 0 0 0 0] diff --git a/cranelift/filetests/filetests/runtests/popcnt.clif b/cranelift/filetests/filetests/runtests/popcnt.clif new file mode 100644 index 000000000000..4f6f7aa9c435 --- /dev/null +++ b/cranelift/filetests/filetests/runtests/popcnt.clif @@ -0,0 +1,100 @@ +test interpret +test run +target aarch64 +target x86_64 +target x86_64 has_popcnt=1 + +function %popcnt_i8(i8) -> i8 { +block0(v0: i8): + v1 = popcnt v0 + return v1 +} +; run: %popcnt_i8(1) == 1 +; run: %popcnt_i8(0x40) == 1 +; run: %popcnt_i8(-1) == 8 +; run: %popcnt_i8(0) == 0 + +; Regression test for issue #3615 +function %inv_popcnt_i8(i8) -> i8 { +block0(v0: i8): + v1 = bnot v0 + v2 = popcnt v1 + return v2 +} +; run: %inv_popcnt_i8(1) == 7 +; run: %inv_popcnt_i8(0x40) == 7 +; run: %inv_popcnt_i8(-1) == 0 +; run: %inv_popcnt_i8(0) == 8 + +function %popcnt_i16(i16) -> i16 { +block0(v0: i16): + v1 = popcnt v0 + return v1 +} +; run: %popcnt_i16(1) == 1 +; run: %popcnt_i16(0x4000) == 1 +; run: %popcnt_i16(-1) == 16 +; run: %popcnt_i16(0) == 0 + +; Regression test for issue #3615 +function %inv_popcnt_i16(i16) -> i16 { +block0(v0: i16): + v1 = bnot v0 + v2 = popcnt v1 + return v2 +} +; run: %inv_popcnt_i16(1) == 15 +; run: %inv_popcnt_i16(0x4000) == 15 +; run: %inv_popcnt_i16(-1) == 0 +; run: %inv_popcnt_i16(0) == 16 + +function %popcnt_i32(i32) -> i32 { +block0(v0: i32): + v1 = popcnt v0 + return v1 +} +; run: %popcnt_i32(1) == 1 +; run: %popcnt_i32(0x40000000) == 1 +; run: %popcnt_i32(-1) == 32 +; run: %popcnt_i32(0) == 0 + +; Regression test for issue #3615 +function %inv_popcnt_i32(i32) -> i32 { +block0(v0: i32): + v1 = bnot v0 + v2 = popcnt v1 + return v2 +} +; run: %inv_popcnt_i32(1) == 31 +; run: %inv_popcnt_i32(0x40000000) == 31 +; run: %inv_popcnt_i32(-1) == 0 +; run: %inv_popcnt_i32(0) == 32 + +function %popcnt_i64(i64) -> i64 { +block0(v0: i64): + v1 = popcnt v0 + return v1 +} +; run: %popcnt_i64(1) == 1 +; run: %popcnt_i64(0x4000000000000000) == 1 +; run: %popcnt_i64(-1) == 64 +; run: %popcnt_i64(0) == 0 + +; Regression test for issue #3615 +function %inv_popcnt_i64(i64) -> i64 { +block0(v0: i64): + v1 = bnot v0 + v2 = popcnt v1 + return v2 +} +; run: %inv_popcnt_i64(1) == 63 +; run: %inv_popcnt_i64(0x4000000000000000) == 63 +; run: %inv_popcnt_i64(-1) == 0 +; run: %inv_popcnt_i64(0) == 64 + +function %popcnt_i8x16(i8x16) -> i8x16 { +block0(v0: i8x16): + v1 = popcnt v0 + return v1 +} +; run: %popcnt_i8x16([1 1 1 1 0x40 0x40 0x40 0x40 0xff 0xff 0xff 0xff 0 0 0 0]) == [1 1 1 1 1 1 1 1 8 8 8 8 0 0 0 0] diff --git a/cranelift/filetests/filetests/runtests/shifts.clif b/cranelift/filetests/filetests/runtests/shifts.clif index c39ff3a0d4b3..5f66d56191a1 100644 --- a/cranelift/filetests/filetests/runtests/shifts.clif +++ b/cranelift/filetests/filetests/runtests/shifts.clif @@ -318,3 +318,25 @@ block0(v0: i32, v1: i8): ; run: %sshr_i32_i8(0x40000000, 32) == 0x40000000 ; run: %sshr_i32_i8(0x40000000, 33) == 0x20000000 ; run: %sshr_i32_i8(0x40000000, 34) == 0x10000000 + +function %rotl_i8_const_37(i8) -> i8 { +block0(v0: i8): + v1 = iconst.i8 37 + v2 = rotl.i8 v0, v1 + return v2 +} + +; run: %rotl_i8_const_37(0x00) == 0x00 +; run: %rotl_i8_const_37(0x01) == 0x20 +; run: %rotl_i8_const_37(0x12) == 0x42 + +function %rotr_i8_const_37(i8) -> i8 { +block0(v0: i8): + v1 = iconst.i8 37 + v2 = rotr.i8 v0, v1 + return v2 +} + +; run: %rotr_i8_const_37(0x00) == 0x00 +; run: %rotr_i8_const_37(0x01) == 0x08 +; run: %rotr_i8_const_37(0x12) == 0x90