[X86] Support promoted ENQCMD, KEYLOCKER and USERMSR

llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h

@@ -140,7 +140,9 @@ enum attributeBits {
   ENUM_ENTRY(IC_EVEX, 1, "requires an EVEX prefix")                            \
   ENUM_ENTRY(IC_EVEX_NF, 2, "requires EVEX and NF prefix")                     \
   ENUM_ENTRY(IC_EVEX_XS, 2, "requires EVEX and the XS prefix")                 \
+  ENUM_ENTRY(IC_EVEX_XS_ADSIZE, 3, "requires EVEX, XS and the ADSIZE prefix")  \
   ENUM_ENTRY(IC_EVEX_XD, 2, "requires EVEX and the XD prefix")                 \
+  ENUM_ENTRY(IC_EVEX_XD_ADSIZE, 3, "requires EVEX, XD and the ADSIZE prefix")  \
   ENUM_ENTRY(IC_EVEX_OPSIZE, 2, "requires EVEX and the OpSize prefix")         \
   ENUM_ENTRY(IC_EVEX_OPSIZE_NF, 3, "requires EVEX, NF and the OpSize prefix")  \
   ENUM_ENTRY(IC_EVEX_OPSIZE_ADSIZE, 3,                                         \

llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp

@@ -941,6 +941,9 @@ static bool readOpcode(struct InternalInstruction *insn) {
     case VEX_LOB_MAP6:
       insn->opcodeType = MAP6;
       return consume(insn, insn->opcode);
+    case VEX_LOB_MAP7:
+      insn->opcodeType = MAP7;
+      return consume(insn, insn->opcode);
     }
   } else if (insn->vectorExtensionType == TYPE_VEX_3B) {
     switch (mmmmmFromVEX2of3(insn->vectorExtensionPrefix[1])) {

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

@@ -5035,14 +5035,18 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
       unsigned Opcode;
       switch (IntNo) {
       default: llvm_unreachable("Impossible intrinsic");
-      case Intrinsic::x86_encodekey128: Opcode = X86::ENCODEKEY128; break;
-      case Intrinsic::x86_encodekey256: Opcode = X86::ENCODEKEY256; break;
+      case Intrinsic::x86_encodekey128:
+        Opcode = GET_EGPR_IF_ENABLED(X86::ENCODEKEY128);
+        break;
+      case Intrinsic::x86_encodekey256:
+        Opcode = GET_EGPR_IF_ENABLED(X86::ENCODEKEY256);
+        break;
       }
 
       SDValue Chain = Node->getOperand(0);
       Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0, Node->getOperand(3),
                                    SDValue());
-      if (Opcode == X86::ENCODEKEY256)
+      if (Opcode == X86::ENCODEKEY256 || Opcode == X86::ENCODEKEY256_EVEX)
         Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1, Node->getOperand(4),
                                      Chain.getValue(1));
 
@@ -5514,7 +5518,6 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
       LoReg = UseMULX ? X86::RDX : X86::RAX;
       HiReg = X86::RDX;
       break;
-#undef GET_EGPR_IF_ENABLED
     }
 
     SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
@@ -6394,17 +6397,18 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
     default:
       llvm_unreachable("Unexpected opcode!");
     case X86ISD::AESENCWIDE128KL:
-      Opcode = X86::AESENCWIDE128KL;
+      Opcode = GET_EGPR_IF_ENABLED(X86::AESENCWIDE128KL);
       break;
     case X86ISD::AESDECWIDE128KL:
-      Opcode = X86::AESDECWIDE128KL;
+      Opcode = GET_EGPR_IF_ENABLED(X86::AESDECWIDE128KL);
       break;
     case X86ISD::AESENCWIDE256KL:
-      Opcode = X86::AESENCWIDE256KL;
+      Opcode = GET_EGPR_IF_ENABLED(X86::AESENCWIDE256KL);
       break;
     case X86ISD::AESDECWIDE256KL:
-      Opcode = X86::AESDECWIDE256KL;
+      Opcode = GET_EGPR_IF_ENABLED(X86::AESDECWIDE256KL);
       break;
+#undef GET_EGPR_IF_ENABLED
     }
 
     SDValue Chain = Node->getOperand(0);

llvm/lib/Target/X86/X86InstrKL.td

@@ -14,61 +14,75 @@
 
 //===----------------------------------------------------------------------===//
 // Key Locker instructions
+class Encodekey<bits<8> opcode, string m>
+  : I<opcode, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), m#"\t{$src, $dst|$dst, $src}", []>,
+    NoCD8, XS;
 
-let SchedRW = [WriteSystem], Predicates = [HasKL] in {
-  let Uses = [XMM0, EAX], Defs = [EFLAGS] in {
+multiclass Aesencdec<string suffix> {
+ def AESENC128KL#suffix : I<0xDC, MRMSrcMem, (outs VR128:$dst),
+                            (ins VR128:$src1, opaquemem:$src2),
+                            "aesenc128kl\t{$src2, $src1|$src1, $src2}",
+                            [(set VR128:$dst, EFLAGS, (X86aesenc128kl VR128:$src1, addr:$src2))]>,
+                          NoCD8, XS;
+ def AESDEC128KL#suffix : I<0xDD, MRMSrcMem, (outs VR128:$dst),
+                            (ins VR128:$src1, opaquemem:$src2),
+                            "aesdec128kl\t{$src2, $src1|$src1, $src2}",
+                            [(set VR128:$dst, EFLAGS, (X86aesdec128kl VR128:$src1, addr:$src2))]>,
+                          NoCD8, XS;
+ def AESENC256KL#suffix : I<0xDE, MRMSrcMem, (outs VR128:$dst),
+                            (ins VR128:$src1, opaquemem:$src2),
+                            "aesenc256kl\t{$src2, $src1|$src1, $src2}",
+                            [(set VR128:$dst, EFLAGS, (X86aesenc256kl VR128:$src1, addr:$src2))]>,
+                          NoCD8, XS;
+ def AESDEC256KL#suffix : I<0xDF, MRMSrcMem, (outs VR128:$dst),
+                            (ins VR128:$src1, opaquemem:$src2),
+                            "aesdec256kl\t{$src2, $src1|$src1, $src2}",
+                            [(set VR128:$dst, EFLAGS, (X86aesdec256kl VR128:$src1, addr:$src2))]>,
+                          NoCD8, XS;
+}
+
+let SchedRW = [WriteSystem] in {
+  let Uses = [XMM0, EAX], Defs = [EFLAGS], Predicates = [HasKL] in {
     def LOADIWKEY : I<0xDC, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2),
                       "loadiwkey\t{$src2, $src1|$src1, $src2}",
                       [(int_x86_loadiwkey XMM0, VR128:$src1, VR128:$src2, EAX)]>, T8, XS;
   }
 
-  let Uses = [XMM0], Defs = [XMM0, XMM1, XMM2, XMM4, XMM5, XMM6, EFLAGS] in {
-    def ENCODEKEY128 : I<0xFA, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
-                         "encodekey128\t{$src, $dst|$dst, $src}", []>, T8, XS;
-  }
+  let Predicates = [HasKL, NoEGPR] in {
+    let Uses = [XMM0], Defs = [XMM0, XMM1, XMM2, XMM4, XMM5, XMM6, EFLAGS] in
+      def ENCODEKEY128 : Encodekey<0xFA, "encodekey128">, T8;
 
-  let Uses = [XMM0, XMM1], Defs = [XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, EFLAGS] in {
-    def ENCODEKEY256 : I<0xFB, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
-                         "encodekey256\t{$src, $dst|$dst, $src}", []>, T8, XS;
-  }
+    let Uses = [XMM0, XMM1], Defs = [XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, EFLAGS] in
+      def ENCODEKEY256 : Encodekey<0xFB, "encodekey256">, T8;
 
-  let Constraints = "$src1 = $dst",
-      Defs = [EFLAGS] in {
-   def AESENC128KL : I<0xDC, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, opaquemem:$src2),
-                       "aesenc128kl\t{$src2, $src1|$src1, $src2}",
-                       [(set VR128:$dst, EFLAGS,
-                         (X86aesenc128kl VR128:$src1, addr:$src2))]>, T8, XS;
+    let Constraints = "$src1 = $dst", Defs = [EFLAGS] in
+      defm "" : Aesencdec<"">, T8;
+  }
 
-   def AESDEC128KL : I<0xDD, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, opaquemem:$src2),
-                       "aesdec128kl\t{$src2, $src1|$src1, $src2}",
-                       [(set VR128:$dst, EFLAGS,
-                         (X86aesdec128kl VR128:$src1, addr:$src2))]>, T8, XS;
+  let Predicates = [HasKL, HasEGPR, In64BitMode] in {
+    let Uses = [XMM0], Defs = [XMM0, XMM1, XMM2, XMM4, XMM5, XMM6, EFLAGS] in
+      def ENCODEKEY128_EVEX : Encodekey<0xDA, "encodekey128">, EVEX, T_MAP4;
 
-   def AESENC256KL : I<0xDE, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, opaquemem:$src2),
-                       "aesenc256kl\t{$src2, $src1|$src1, $src2}",
-                       [(set VR128:$dst, EFLAGS,
-                         (X86aesenc256kl VR128:$src1, addr:$src2))]>, T8, XS;
+    let Uses = [XMM0, XMM1], Defs = [XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, EFLAGS] in
+      def ENCODEKEY256_EVEX : Encodekey<0xDB, "encodekey256">, EVEX, T_MAP4;
 
-   def AESDEC256KL : I<0xDF, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, opaquemem:$src2),
-                       "aesdec256kl\t{$src2, $src1|$src1, $src2}",
-                       [(set VR128:$dst, EFLAGS,
-                         (X86aesdec256kl VR128:$src1, addr:$src2))]>, T8, XS;
+    let Constraints = "$src1 = $dst", Defs = [EFLAGS] in
+      defm "" : Aesencdec<"_EVEX">, EVEX, T_MAP4;
   }
+} // SchedRW
 
-} // SchedRW, Predicates
+multiclass Aesencdecwide<string suffix> {
+  def AESENCWIDE128KL#suffix : I<0xD8, MRM0m, (outs), (ins opaquemem:$src), "aesencwide128kl\t$src", []>, NoCD8, XS;
+  def AESDECWIDE128KL#suffix : I<0xD8, MRM1m, (outs), (ins opaquemem:$src), "aesdecwide128kl\t$src", []>, NoCD8, XS;
+  def AESENCWIDE256KL#suffix : I<0xD8, MRM2m, (outs), (ins opaquemem:$src), "aesencwide256kl\t$src", []>, NoCD8, XS;
+  def AESDECWIDE256KL#suffix : I<0xD8, MRM3m, (outs), (ins opaquemem:$src), "aesdecwide256kl\t$src", []>, NoCD8, XS;
+}
 
-let SchedRW = [WriteSystem], Predicates = [HasWIDEKL] in {
-  let Uses = [XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7],
-      Defs = [EFLAGS, XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7],
-      mayLoad = 1 in {
-    def AESENCWIDE128KL : I<0xD8, MRM0m, (outs), (ins opaquemem:$src),
-                            "aesencwide128kl\t$src", []>, T8, XS;
-    def AESDECWIDE128KL : I<0xD8, MRM1m, (outs), (ins opaquemem:$src),
-                            "aesdecwide128kl\t$src", []>, T8, XS;
-    def AESENCWIDE256KL : I<0xD8, MRM2m, (outs), (ins opaquemem:$src),
-                            "aesencwide256kl\t$src", []>, T8, XS;
-    def AESDECWIDE256KL : I<0xD8, MRM3m, (outs), (ins opaquemem:$src),
-                            "aesdecwide256kl\t$src", []>, T8, XS;
-  }
+let SchedRW = [WriteSystem], Uses = [XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7],
+    Defs = [EFLAGS, XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7], mayLoad = 1 in {
+  let Predicates = [HasWIDEKL, NoEGPR] in
+    defm "" : Aesencdecwide<"">, T8;
 
-} // SchedRW, Predicates
+  let Predicates = [HasWIDEKL, HasEGPR, In64BitMode] in
+    defm "" : Aesencdecwide<"_EVEX">, EVEX, T_MAP4;
+} // SchedRW

llvm/lib/Target/X86/X86InstrMisc.td

@@ -1557,32 +1557,39 @@ def MOVDIR64B64_EVEX : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$
 //===----------------------------------------------------------------------===//
 // ENQCMD/S - Enqueue 64-byte command as user with 64-byte write atomicity
 //
+multiclass Enqcmds<string suffix> {
+  def ENQCMD32#suffix : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem_GR32:$src),
+                          "enqcmd\t{$src, $dst|$dst, $src}",
+                          [(set EFLAGS, (X86enqcmd GR32:$dst, addr:$src))]>,
+                        NoCD8, XD, AdSize32;
+  def ENQCMD64#suffix : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$src),
+                          "enqcmd\t{$src, $dst|$dst, $src}",
+                          [(set EFLAGS, (X86enqcmd GR64:$dst, addr:$src))]>,
+                        NoCD8, XD, AdSize64;
+
+  def ENQCMDS32#suffix : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem_GR32:$src),
+                           "enqcmds\t{$src, $dst|$dst, $src}",
+                           [(set EFLAGS, (X86enqcmds GR32:$dst, addr:$src))]>,
+                         NoCD8, XS, AdSize32;
+  def ENQCMDS64#suffix : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$src),
+                          "enqcmds\t{$src, $dst|$dst, $src}",
+                          [(set EFLAGS, (X86enqcmds GR64:$dst, addr:$src))]>,
+                         NoCD8, XS, AdSize64;
+}
+
 let SchedRW = [WriteStore], Defs = [EFLAGS] in {
   def ENQCMD16 : I<0xF8, MRMSrcMem, (outs), (ins GR16:$dst, i512mem_GR16:$src),
-                 "enqcmd\t{$src, $dst|$dst, $src}",
-                 [(set EFLAGS, (X86enqcmd GR16:$dst, addr:$src))]>,
+                   "enqcmd\t{$src, $dst|$dst, $src}",
+                   [(set EFLAGS, (X86enqcmd GR16:$dst, addr:$src))]>,
                  T8, XD, AdSize16, Requires<[HasENQCMD, Not64BitMode]>;
-  def ENQCMD32 : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem_GR32:$src),
-                 "enqcmd\t{$src, $dst|$dst, $src}",
-                 [(set EFLAGS, (X86enqcmd GR32:$dst, addr:$src))]>,
-                 T8, XD, AdSize32, Requires<[HasENQCMD]>;
-  def ENQCMD64 : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$src),
-                 "enqcmd\t{$src, $dst|$dst, $src}",
-                 [(set EFLAGS, (X86enqcmd GR64:$dst, addr:$src))]>,
-                 T8, XD, AdSize64, Requires<[HasENQCMD, In64BitMode]>;
-
   def ENQCMDS16 : I<0xF8, MRMSrcMem, (outs), (ins GR16:$dst, i512mem_GR16:$src),
-                 "enqcmds\t{$src, $dst|$dst, $src}",
-                 [(set EFLAGS, (X86enqcmds GR16:$dst, addr:$src))]>,
-                 T8, XS, AdSize16, Requires<[HasENQCMD, Not64BitMode]>;
-  def ENQCMDS32 : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem_GR32:$src),
-                 "enqcmds\t{$src, $dst|$dst, $src}",
-                 [(set EFLAGS, (X86enqcmds GR32:$dst, addr:$src))]>,
-                 T8, XS, AdSize32, Requires<[HasENQCMD]>;
-  def ENQCMDS64 : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$src),
-                 "enqcmds\t{$src, $dst|$dst, $src}",
-                 [(set EFLAGS, (X86enqcmds GR64:$dst, addr:$src))]>,
-                 T8, XS, AdSize64, Requires<[HasENQCMD, In64BitMode]>;
+                    "enqcmds\t{$src, $dst|$dst, $src}",
+                    [(set EFLAGS, (X86enqcmds GR16:$dst, addr:$src))]>,
+                  T8, XS, AdSize16, Requires<[HasENQCMD, Not64BitMode]>;
+
+  defm "" : Enqcmds<"">, T8, Requires<[HasENQCMD, NoEGPR]>;
+  defm "" : Enqcmds<"_EVEX">, EVEX, T_MAP4, Requires<[HasENQCMD, HasEGPR, In64BitMode]>;
+
 }
 
 //===----------------------------------------------------------------------===//

llvm/lib/Target/X86/X86InstrSystem.td

@@ -436,22 +436,35 @@ def WRMSRLIST : I<0x01, MRM_C6, (outs), (ins), "wrmsrlist", []>, TB, XS;
 def RDMSRLIST : I<0x01, MRM_C6, (outs), (ins), "rdmsrlist", []>, TB, XD;
 }
 
-let Predicates = [HasUSERMSR], mayLoad = 1 in {
-  def URDMSRrr : I<0xf8, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src),
-                "urdmsr\t{$src, $dst|$dst, $src}",
-                [(set GR64:$dst, (int_x86_urdmsr GR64:$src))]>, T8, XD;
-  def URDMSRri : Ii32<0xf8, MRM0r, (outs GR64:$dst), (ins i64i32imm:$imm),
-                "urdmsr\t{$imm, $dst|$dst, $imm}",
-                [(set GR64:$dst, (int_x86_urdmsr i64immSExt32_su:$imm))]>, T_MAP7, XD, VEX;
+multiclass Urdwrmsr<Map rrmap, string suffix> {
+  let mayLoad = 1 in {
+    let OpMap = rrmap in
+    def URDMSRrr#suffix : I<0xf8, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src),
+                            "urdmsr\t{$src, $dst|$dst, $src}",
+                            [(set GR64:$dst, (int_x86_urdmsr GR64:$src))]>, XD, NoCD8;
+    def URDMSRri#suffix  : Ii32<0xf8, MRM0r, (outs GR64:$dst), (ins i64i32imm:$imm),
+                                "urdmsr\t{$imm, $dst|$dst, $imm}",
+                                [(set GR64:$dst, (int_x86_urdmsr i64immSExt32_su:$imm))]>,
+                           T_MAP7, VEX, XD, NoCD8;
 }
-let Predicates = [HasUSERMSR], mayStore = 1 in {
-  def UWRMSRrr : I<0xf8, MRMSrcReg, (outs), (ins GR64:$src1, GR64:$src2),
-                "uwrmsr\t{$src2, $src1|$src1, $src2}",
-                [(int_x86_uwrmsr GR64:$src1, GR64:$src2)]>, T8, XS;
-  def UWRMSRir : Ii32<0xf8, MRM0r, (outs), (ins GR64:$src, i64i32imm:$imm),
-                "uwrmsr\t{$src, $imm|$imm, $src}",
-                [(int_x86_uwrmsr i64immSExt32_su:$imm, GR64:$src)]>, T_MAP7, XS, VEX;
+  let mayStore = 1 in {
+    let OpMap = rrmap in
+    def UWRMSRrr#suffix  : I<0xf8, MRMSrcReg, (outs), (ins GR64:$src1, GR64:$src2),
+                             "uwrmsr\t{$src2, $src1|$src1, $src2}",
+                             [(int_x86_uwrmsr GR64:$src1, GR64:$src2)]>, XS, NoCD8;
+    def UWRMSRir#suffix  : Ii32<0xf8, MRM0r, (outs), (ins GR64:$src, i64i32imm:$imm),
+                                "uwrmsr\t{$src, $imm|$imm, $src}",
+                                [(int_x86_uwrmsr i64immSExt32_su:$imm, GR64:$src)]>,
+                           T_MAP7, VEX, XS, NoCD8;
+  }
 }
+
+let Predicates = [HasUSERMSR, NoEGPR] in
+  defm "" : Urdwrmsr<T8, "">;
+
+let Predicates = [HasUSERMSR, HasEGPR, In64BitMode] in
+  defm "" : Urdwrmsr<T_MAP4, "_EVEX">, EVEX;
+
 let Defs = [RAX, RDX], Uses = [ECX] in
 def RDPMC : I<0x33, RawFrm, (outs), (ins), "rdpmc", []>, TB;
 

llvm/test/CodeGen/X86/enqcmd-intrinsics.ll

@@ -2,6 +2,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+enqcmd | FileCheck %s --check-prefix=X64
 ; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+enqcmd | FileCheck %s --check-prefix=X86
 ; RUN: llc < %s -mtriple=x86_64-linux-gnux32 -mattr=+enqcmd | FileCheck %s --check-prefix=X32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+enqcmd,+egpr --show-mc-encoding | FileCheck %s --check-prefix=EGPR
 
 define i8 @test_enqcmd(ptr %dst, ptr %src) {
 ; X64-LABEL: test_enqcmd:
@@ -23,6 +24,12 @@ define i8 @test_enqcmd(ptr %dst, ptr %src) {
 ; X32-NEXT:    enqcmd (%esi), %edi
 ; X32-NEXT:    sete %al
 ; X32-NEXT:    retq
+;
+; EGPR-LABEL: test_enqcmd:
+; EGPR:       # %bb.0: # %entry
+; EGPR-NEXT:    enqcmd (%rsi), %rdi # EVEX TO LEGACY Compression encoding: [0xf2,0x0f,0x38,0xf8,0x3e]
+; EGPR-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
+; EGPR-NEXT:    retq # encoding: [0xc3]
 entry:
 
 
@@ -50,6 +57,12 @@ define i8 @test_enqcmds(ptr %dst, ptr %src) {
 ; X32-NEXT:    enqcmds (%esi), %edi
 ; X32-NEXT:    sete %al
 ; X32-NEXT:    retq
+;
+; EGPR-LABEL: test_enqcmds:
+; EGPR:       # %bb.0: # %entry
+; EGPR-NEXT:    enqcmds (%rsi), %rdi # EVEX TO LEGACY Compression encoding: [0xf3,0x0f,0x38,0xf8,0x3e]
+; EGPR-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
+; EGPR-NEXT:    retq # encoding: [0xc3]
 entry: